Source code for google.appengine.ext.appstats.ui

#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#




"""Web-based User Interface for appstats.

This is a simple set of webapp-based request handlers that display the
collected statistics and let you drill down on the information in
various ways.

Template files are in the templates/ subdirectory.  Static files are
in the static/ subdirectory.

The templates are written to work with either Django 0.96 or Django
1.0, and most likely they also work with Django 1.1.
"""



import cgi
import cStringIO
import email.Utils
import logging
import mimetypes
import os
import re
import sys
import time
import traceback

from google.appengine.api import users
from google.appengine.ext import admin
from google.appengine.ext import webapp
from google.appengine.ext.appstats import datamodel_pb
from google.appengine.ext.appstats import recording
from google.appengine.ext.webapp import _template
from google.appengine.ext.webapp import util

DEBUG = recording.config.DEBUG


def _add_billed_ops_to_map(billed_ops_map, billed_ops_list):
  """Adds the BilledOpProto objects to the given map.

  The map is from BilledOpProto.op to the pb.

  Args:
    billed_ops_map: The map to populate.
    billed_ops_list: List containing the BilledOpProtos to add to the map.
  """
  for billed_op in billed_ops_list:
    if billed_op.op() not in billed_ops_map:
      update_me = datamodel_pb.BilledOpProto()
      update_me.set_op(billed_op.op())
      update_me.set_num_ops(0)
      billed_ops_map[billed_op.op()] = update_me
    update_me = billed_ops_map[billed_op.op()]
    update_me.set_num_ops(update_me.num_ops() + billed_op.num_ops())


def _billed_ops_to_str(billed_ops):
  """Builds a string representation of a list of BilledOpProto."""
  ops_as_strs = []
  for op in billed_ops:
    op_name = datamodel_pb.BilledOpProto.BilledOp_Name(op.op())
    ops_as_strs.append('%s:%s' % (op_name, op.num_ops()))
  return ', '.join(ops_as_strs)


def _as_percentage_of(cost_micropennies, total_cost_micropennies):
  """The cost as a percentage of the total cost, rounded to hundredths."""
  if total_cost_micropennies == 0:
    return 0
  return round((float(cost_micropennies) / float(total_cost_micropennies))
               * 100, 1)


[docs]def render(tmplname, data): """Helper function to render a template.""" here = os.path.dirname(__file__) tmpl = os.path.join(here, 'templates', tmplname) data['env'] = os.environ data['shell_ok'] = recording.config.SHELL_OK data['multithread'] = os.getenv('wsgi.multithread') try: return _template.render(tmpl, data) except Exception, err: logging.exception('Failed to render %s', tmpl) return 'Problematic template %s: %s' % (tmplname, err)
[docs]class AllStatsInfo(object): """AllStats data.""" def __init__(self, calls, cost, billed_ops): self.calls = calls self.cost = cost self.billed_ops = billed_ops
[docs]class PathStatsInfo(object): """PathStats data.""" def __init__(self, cost, billed_ops, num_requests, most_recent_requests): self.cost = cost self.billed_ops = billed_ops self.num_requests = num_requests self.most_recent_requests = most_recent_requests
[docs]class PivotInfo(object): """Pivot data. The name attribute can be an rpc or a path.""" def __init__(self, name, calls, cost, billed_ops, cost_pct): self.name = name self.calls = calls self.cost = cost self.billed_ops = billed_ops self.cost_pct = cost_pct
[docs] def to_list(self): """Convert to a list with values in the locations expected by the ui.""" return [self.name, self.calls, self.cost, self.billed_ops, self.cost_pct]
[docs] @classmethod def from_list(cls, values): return cls(values[0], values[1], values[2], values[3], values[4])
[docs]class SummaryHandler(webapp.RequestHandler): """Request handler for the main stats page (/stats/)."""
[docs] def get(self): recording.dont_record() if not self.request.path.endswith('/'): self.redirect(self.request.path + '/') return summaries = recording.load_summary_protos() data = self._get_summary_data(summaries) self.response.out.write(render('main.html', data))
def _get_summary_data(self, summaries): """Extract statistics from summaries.""" allstats = {} pathstats = {} pivot_path_rpc = {} pivot_rpc_path = {} total_cost_micropennies = 0 summaries = sorted(summaries, key=lambda x: (-x.start_timestamp_milliseconds())) for index, summary in enumerate(summaries): path_key = recording.config.extract_key(summary) if path_key not in pathstats: pathstats[path_key] = PathStatsInfo(0, {}, 1, [index+1]) else: pathstats_info = pathstats[path_key] pathstats_info.num_requests += 1 if len(pathstats_info.most_recent_requests) > 10: if pathstats_info.most_recent_requests[-1]: pathstats_info.most_recent_requests.append(0) else: pathstats_info.most_recent_requests.append(index+1) if path_key not in pivot_path_rpc: pivot_path_rpc[path_key] = {} for x in summary.rpc_stats_list(): rpc_key = x.service_call_name() total_calls = x.total_amount_of_calls() cost_micropennies = x.total_cost_of_calls_microdollars() total_cost_micropennies += cost_micropennies pathstats[path_key].cost += cost_micropennies _add_billed_ops_to_map(pathstats[path_key].billed_ops, x.total_billed_ops_list()) if rpc_key in allstats: allstats[rpc_key].calls += total_calls allstats[rpc_key].cost += cost_micropennies else: allstats[rpc_key] = AllStatsInfo(total_calls, cost_micropennies, {}) _add_billed_ops_to_map( allstats[rpc_key].billed_ops, x.total_billed_ops_list()) if rpc_key not in pivot_path_rpc[path_key]: pivot_path_rpc[path_key][rpc_key] = PivotInfo(rpc_key, 0, 0, {}, 0) pivot_path_rpc[path_key][rpc_key].calls += total_calls pivot_path_rpc[path_key][rpc_key].cost += cost_micropennies _add_billed_ops_to_map(pivot_path_rpc[path_key][rpc_key].billed_ops, x.total_billed_ops_list()) if rpc_key not in pivot_rpc_path: pivot_rpc_path[rpc_key] = {} if path_key not in pivot_rpc_path[rpc_key]: pivot_rpc_path[rpc_key][path_key] = PivotInfo(path_key, 0, 0, {}, 0) pivot_rpc_path[rpc_key][path_key].calls += total_calls pivot_rpc_path[rpc_key][path_key].cost += cost_micropennies _add_billed_ops_to_map(pivot_rpc_path[rpc_key][path_key].billed_ops, x.total_billed_ops_list()) allstats_by_count = [] for k, v in allstats.iteritems(): for path_vals in pivot_rpc_path[k].itervalues(): path_vals.billed_ops = _billed_ops_to_str( path_vals.billed_ops.itervalues()) path_vals.cost_pct = _as_percentage_of( path_vals.cost, total_cost_micropennies) pivot = sorted(pivot_rpc_path[k].itervalues(), key=lambda x: (-x.calls, x.name)) allstats_by_count.append(( k, v.calls, v.cost, _billed_ops_to_str(v.billed_ops.itervalues()), _as_percentage_of(v.cost, total_cost_micropennies), [x.to_list() for x in pivot])) allstats_by_count.sort(key=lambda x: (-x[1], x[0])) pathstats_by_count = [] for path_key, pathstats_info in pathstats.iteritems(): rpc_count = 0 for rpc_vals in pivot_path_rpc[path_key].itervalues(): rpc_vals.billed_ops = _billed_ops_to_str( rpc_vals.billed_ops.itervalues()) rpc_vals.cost_pct = _as_percentage_of( rpc_vals.cost, total_cost_micropennies) rpc_count += rpc_vals.calls pivot = sorted(pivot_path_rpc[path_key].itervalues(), key=lambda x: (-x.calls, x.name)) pathstats_by_count.append(( path_key, rpc_count, pathstats_info.cost, _billed_ops_to_str(pathstats_info.billed_ops.itervalues()), _as_percentage_of(pathstats_info.cost, total_cost_micropennies), pathstats_info.num_requests, pathstats_info.most_recent_requests, [x.to_list() for x in pivot])) pathstats_by_count.sort(key=lambda x: (-x[1], -x[5], x[0])) return {'requests': summaries, 'allstats_by_count': allstats_by_count, 'pathstats_by_count': pathstats_by_count, }
[docs]class DetailsHandler(webapp.RequestHandler): """Request handler for the details page (/stats/details)."""
[docs] def get(self): recording.dont_record() time_key = self.request.get('time') timestamp = None record = None if time_key: try: timestamp = int(time_key) * 0.001 except Exception: pass if timestamp: record = recording.load_full_proto(timestamp) render_record(self.response, record, './file')
[docs]def render_record(response, record, file_url=None, extra_data=None): """Render an appstats record in detail. This is a minor refactoring of DetailsHandler to support an offline tool for analyzing Appstats data and to allow that tool to call the original Appstats detailed record visualization. Since the offline tool may read Appstats records from other sources (e.g., a downloaded file), we are moving the logic of DetailsHandler related to processing and visualizing individual Appstats records to this function. This function may now be called from outside this file. Args: response: An instance of the webapp response class representing data to be sent in response to a web request. record: A RequestStatProto which contains detailed Appstats recording for an individual request. file_url: Indicates the URL to be used to follow links to files in application source code. A default value of 'None' indicates that links to files in source code will not be shown. extra_data: Optional dict of additional parameters for template. """ data = {} if extra_data is not None: data.update(extra_data) if record is None: if extra_data is None: response.set_status(404) response.out.write(render('details.html', data)) return data.update(get_details_data(record, file_url)) response.out.write(render('details.html', data))
[docs]def get_details_data(record, file_url=None): """ Calculate detailed appstats data for a single request. Args: record: A RequestStatProto which contains detailed Appstats recording for an individual request. file_url: Indicates the URL to be used to follow links to files in application source code. A default value of 'None' indicates that links to files in source code will not be shown. Returns: A dictionary containing detailed appstats data for a single request. """ rpcstats_map = {} for rpc_stat in record.individual_stats_list(): key = rpc_stat.service_call_name() count, real, api, rpc_cost_micropennies, billed_ops = rpcstats_map.get( key, (0, 0, 0, 0, {})) count += 1 real += rpc_stat.duration_milliseconds() api += rpc_stat.api_mcycles() rpc_cost_micropennies += rpc_stat.call_cost_microdollars() _add_billed_ops_to_map(billed_ops, rpc_stat.billed_ops_list()) rpcstats_map[key] = (count, real, api, rpc_cost_micropennies, billed_ops) rpcstats_by_count = [ (name, count, real, recording.mcycles_to_msecs(api), rpc_cost_micropennies, _billed_ops_to_str(billed_ops.itervalues())) for name, (count, real, api, rpc_cost_micropennies, billed_ops) in rpcstats_map.iteritems()] rpcstats_by_count.sort(key=lambda x: -x[1]) real_total = 0 api_total_mcycles = 0 for i, rpc_stat in enumerate(record.individual_stats_list()): real_total += rpc_stat.duration_milliseconds() api_total_mcycles += rpc_stat.api_mcycles() api_total = recording.mcycles_to_msecs(api_total_mcycles) return {'sys': sys, 'record': record, 'rpcstats_by_count': rpcstats_by_count, 'real_total': real_total, 'api_total': api_total, 'file_url': file_url, }
[docs]class ShellHandler(webapp.RequestHandler): """Request handler for interactive shell. This is like /_ah/admin/interactive, but with Appstats output integrated. GET displays a form; POST runs some code and displays its output + stats. """ def _check_access(self): if recording.config.SHELL_OK: return True self.response.set_status(403) self.response.out.write('You must enable this feature by setting ' 'appstats_SHELL_OK = True in appengine_config.py') return False
[docs] def get(self): recording.dont_record() if not self._check_access(): return script = self.request.get('script', recording.config.DEFAULT_SCRIPT) extra_data = {'is_shell': True, 'script': script, 'xsrf_token': admin.get_xsrf_token(), } render_record(self.response, None, './file', extra_data)
@admin.xsrf_required def post(self): recording.dont_record() if not self._check_access(): return recorder = recording.Recorder(os.environ) recording.recorder_proxy.set_for_current_request(recorder) script = self.request.get('script', '').replace('\r\n', '\n') output, errors = self.execute_script(script) recording.recorder_proxy.clear_for_current_request() recorder.record_http_status(0) recorder.save() record = recorder.get_full_proto() extra_data = {'is_shell': True, 'script': script, 'output': output, 'errors': errors, 'time_key': int(recorder.start_timestamp * 1000), 'xsrf_token': admin.get_xsrf_token(), } render_record(self.response, record, './file', extra_data)
[docs] def execute_script(self, script): save_stdout = sys.stdout save_stderr = sys.stderr new_stdout = cStringIO.StringIO() new_stderr = cStringIO.StringIO() try: sys.stdout = new_stdout sys.stderr = new_stderr exec(script, {}) except BaseException: traceback.print_exc() finally: sys.stdout = save_stdout sys.stderr = save_stderr return new_stdout.getvalue(), new_stderr.getvalue()
[docs]class FileHandler(webapp.RequestHandler): """Request handler for displaying any text file in the system. NOTE: This gives any admin of your app full access to your source code. """
[docs] def get(self): recording.dont_record() lineno = self.request.get('n') try: lineno = int(lineno) except: lineno = 0 filename = self.request.get('f') or '' orig_filename = filename match = re.match('<path\[(\d+)\]>(.*)', filename) if match: index, tail = match.groups() index = int(index) if index < len(sys.path): filename = sys.path[index] + tail try: fp = open(filename) except IOError, err: self.response.out.write('<h1>IOError</h1><pre>%s</pre>' % cgi.escape(str(err))) self.response.set_status(404) else: try: data = {'fp': fp, 'filename': filename, 'orig_filename': orig_filename, 'lineno': lineno, } self.response.out.write(render('file.html', data)) finally: fp.close()
[docs]class StaticHandler(webapp.RequestHandler): """Request handler to serve static files. Only files directory in the static subdirectory are rendered this way (no subdirectories). """
[docs] def get(self): recording.dont_record() here = os.path.dirname(__file__) fn = self.request.path i = fn.rfind('/') fn = fn[i+1:] fn = os.path.join(here, 'static', fn) ctype, encoding = mimetypes.guess_type(fn) assert ctype and '/' in ctype, repr(ctype) expiry = 3600 expiration = email.Utils.formatdate(time.time() + expiry, usegmt=True) fp = open(fn, 'rb') try: self.response.out.write(fp.read()) finally: fp.close() self.response.headers['Content-type'] = ctype self.response.headers['Cache-Control'] = 'public, max-age=expiry' self.response.headers['Expires'] = expiration
URLMAP = [ ('.*/details', DetailsHandler), ('.*/shell', ShellHandler), ('.*/file', FileHandler), ('.*/static/.*', StaticHandler), ('.*', SummaryHandler), ]
[docs]class AuthCheckMiddleware(object): """Middleware which conducts an auth check.""" def __init__(self, application): self._application = application def __call__(self, environ, start_response): if not environ.get('SERVER_SOFTWARE', '').startswith('Dev'): if not users.is_current_user_admin(): if users.get_current_user() is None: start_response('302 Found', [('Location', users.create_login_url(os.getenv('PATH_INFO', '')))]) return [] else: start_response('403 Forbidden', []) return ['Forbidden\n'] return self._application(environ, start_response)
app = AuthCheckMiddleware(webapp.WSGIApplication(URLMAP, debug=DEBUG))
[docs]def main(): """Main program. Run the auth checking middleware wrapped WSGIApplication.""" util.run_bare_wsgi_app(app)
if __name__ == '__main__': main()