#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""A library for managing flags-like configuration that update dynamically.
"""
import logging
import os
import re
import time
if os.environ.get('APPENGINE_RUNTIME') == 'python27':
from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.api import validation
from google.appengine.api import yaml_object
else:
try:
from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.api import validation
from google.appengine.api import yaml_object
except:
from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.ext import validation
from google.appengine.ext import yaml_object
DATASTORE_DEADLINE = 1.5
RESERVED_MARKER = 'ah__conf__'
NAMESPACE = '_' + RESERVED_MARKER
CONFIG_KIND = '_AppEngine_Config'
ACTIVE_KEY_NAME = 'active'
FILENAMES = ['conf.yaml', 'conf.yml']
PARAMETERS = 'parameters'
PARAMETER_NAME_REGEX = '[a-zA-Z][a-zA-Z0-9_]*'
_cached_config = None
[docs]class Config(db.Expando):
"""The representation of a config in the datastore and memcache."""
ah__conf__version = db.IntegerProperty(default=0, required=True)
[docs] @classmethod
def kind(cls):
"""Override the kind name to prevent collisions with users."""
return CONFIG_KIND
[docs] def ah__conf__load_from_yaml(self, parsed_config):
"""Loads all the params from a YAMLConfiguration into expando fields.
We set these expando properties with a special name prefix 'p_' to
keep them separate from the static attributes of Config. That way we
don't have to check elsewhere to make sure the user doesn't stomp on
our built in properties.
Args:
parse_config: A YAMLConfiguration.
"""
for key, value in parsed_config.parameters.iteritems():
setattr(self, key, value)
class _ValidParameterName(validation.Validator):
"""Validator to check if a value is a valid config parameter name.
We only allow valid python attribute names without leading underscores
that also do not collide with reserved words in the datastore models.
"""
def __init__(self):
self.regex = validation.Regex(PARAMETER_NAME_REGEX)
def Validate(self, value, key):
"""Check that all parameter names are valid.
This is used as a validator when parsing conf.yaml.
Args:
value: the value to check.
key: A description of the context for which this value is being
validated.
Returns:
The validated value.
"""
value = self.regex.Validate(value, key)
try:
db.check_reserved_word(value)
except db.ReservedWordError:
raise validation.ValidationError(
'The config parameter name %.100r is reserved by db.Model see: '
'https://developers.google.com/appengine/docs/python/datastore/'
'modelclass#Disallowed_Property_Names for details.' % value)
if value.startswith(RESERVED_MARKER):
raise validation.ValidationError(
'The config parameter name %.100r is reserved, as are all names '
'beginning with \'%s\', please choose a different name.' % (
value, RESERVED_MARKER))
return value
class _Scalar(validation.Validator):
"""Validator to check if a value is a simple scalar type.
We only allow scalars that are well supported by both the datastore and YAML.
"""
ALLOWED_PARAMETER_VALUE_TYPES = frozenset(
[bool, int, long, float, str, unicode])
def Validate(self, value, key):
"""Check that all parameters are scalar values.
This is used as a validator when parsing conf.yaml
Args:
value: the value to check.
key: the name of parameter corresponding to this value.
Returns:
We just return value unchanged.
"""
if type(value) not in self.ALLOWED_PARAMETER_VALUE_TYPES:
raise validation.ValidationError(
'Expected scalar value for parameter: %s, but found %.100r which '
'is type %s' % (key, value, type(value).__name__))
return value
class _ParameterDict(validation.ValidatedDict):
"""This class validates the parameters dictionary in YAMLConfiguration.
Keys must look like non-private python identifiers and values
must be a supported scalar. See the class comment for YAMLConfiguration.
"""
KEY_VALIDATOR = _ValidParameterName()
VALUE_VALIDATOR = _Scalar()
[docs]class YAMLConfiguration(validation.Validated):
"""This class describes the structure of a conf.yaml file.
At the top level the file should have a params attribue which is a mapping
from strings to scalars. For example:
parameters:
background_color: 'red'
message_size: 1024
boolean_valued_param: true
"""
ATTRIBUTES = {PARAMETERS: _ParameterDict}
[docs]def LoadSingleConf(stream):
"""Load a conf.yaml file or string and return a YAMLConfiguration object.
Args:
stream: a file object corresponding to a conf.yaml file, or its contents
as a string.
Returns:
A YAMLConfiguration instance
"""
return yaml_object.BuildSingleObject(YAMLConfiguration, stream)
def _find_yaml_path():
"""Traverse directory trees to find conf.yaml file.
Begins with the current working direcotry and then moves up the
directory structure until the file is found..
Returns:
the path of conf.yaml file or None if not found.
"""
current, last = os.getcwd(), None
while current != last:
for yaml_name in FILENAMES:
yaml_path = os.path.join(current, yaml_name)
if os.path.exists(yaml_path):
return yaml_path
last = current
current, last = os.path.dirname(current), current
return None
def _fetch_from_local_file(pathfinder=_find_yaml_path, fileopener=open):
"""Get the configuration that was uploaded with this version.
Args:
pathfinder: a callable to use for finding the path of the conf.yaml
file. This is only for use in testing.
fileopener: a callable to use for opening a named file. This is
only for use in testing.
Returns:
A config class instance for the options that were uploaded. If there
is no config file, return None
"""
yaml_path = pathfinder()
if yaml_path:
config = Config()
config.ah__conf__load_from_yaml(LoadSingleConf(fileopener(yaml_path)))
logging.debug('Loaded conf parameters from conf.yaml.')
return config
return None
def _get_active_config_key(app_version):
"""Generate the key for the active config record belonging to app_version.
Args:
app_version: the major version you want configuration data for.
Returns:
The key for the active Config record for the given app_version.
"""
return db.Key.from_path(
CONFIG_KIND,
'%s/%s' % (app_version, ACTIVE_KEY_NAME),
namespace=NAMESPACE)
def _fetch_latest_from_datastore(app_version):
"""Get the latest configuration data for this app-version from the datastore.
Args:
app_version: the major version you want configuration data for.
Side Effects:
We populate memcache with whatever we find in the datastore.
Returns:
A config class instance for most recently set options or None if the
query could not complete due to a datastore exception.
"""
rpc = db.create_rpc(deadline=DATASTORE_DEADLINE,
read_policy=db.EVENTUAL_CONSISTENCY)
key = _get_active_config_key(app_version)
config = None
try:
config = Config.get(key, rpc=rpc)
logging.debug('Loaded most recent conf data from datastore.')
except:
logging.warning('Tried but failed to fetch latest conf data from the '
'datastore.')
if config:
memcache.set(app_version, db.model_to_protobuf(config).Encode(),
namespace=NAMESPACE)
logging.debug('Wrote most recent conf data into memcache.')
return config
def _fetch_latest_from_memcache(app_version):
"""Get the latest configuration data for this app-version from memcache.
Args:
app_version: the major version you want configuration data for.
Returns:
A Config class instance for most recently set options or None if none
could be found in memcache.
"""
proto_string = memcache.get(app_version, namespace=NAMESPACE)
if proto_string:
logging.debug('Loaded most recent conf data from memcache.')
return db.model_from_protobuf(proto_string)
logging.debug('Tried to load conf data from memcache, but found nothing.')
return None
def _inspect_environment():
"""Return relevant information from the cgi environment.
This is mostly split out to simplify testing.
Returns:
A tuple: (app_version, conf_version, development)
app_version: the major version of the current application.
conf_version: the current configuration version.
development: a boolean, True if we're running under devappserver.
"""
app_version = os.environ['CURRENT_VERSION_ID'].rsplit('.', 1)[0]
conf_version = int(os.environ.get('CURRENT_CONFIGURATION_VERSION', '0'))
development = os.environ.get('SERVER_SOFTWARE', '').startswith('Development/')
return (app_version, conf_version, development)
[docs]def refresh():
"""Update the local config cache from memcache/datastore.
Normally configuration parameters are only refreshed at the start of a
new request. If you have a very long running request, or you just need
the freshest data for some reason, you can call this function to force
a refresh.
"""
app_version, _, _ = _inspect_environment()
global _cached_config
new_config = _fetch_latest_from_memcache(app_version)
if not new_config:
new_config = _fetch_latest_from_datastore(app_version)
if new_config:
_cached_config = new_config
def _new_request():
"""Test if this is the first call to this function in the current request.
This function will return True exactly once for each request
Subsequent calls in the same request will return False.
Returns:
True if this is the first call in a given request, False otherwise.
"""
if RESERVED_MARKER in os.environ:
return False
os.environ[RESERVED_MARKER] = RESERVED_MARKER
return True
def _get_config():
"""Check if the current cached config is stale, and if so update it."""
app_version, current_config_version, development = _inspect_environment()
global _cached_config
if (development and _new_request()) or not _cached_config:
_cached_config = _fetch_from_local_file() or Config()
if _cached_config.ah__conf__version < current_config_version:
newconfig = _fetch_latest_from_memcache(app_version)
if not newconfig or newconfig.ah__conf__version < current_config_version:
newconfig = _fetch_latest_from_datastore(app_version)
_cached_config = newconfig or _cached_config
return _cached_config
[docs]def get(name, default=None):
"""Get the value of a configuration parameter.
This function is guaranteed to return the same value for every call
during a single request.
Args:
name: The name of the configuration parameter you want a value for.
default: A default value to return if the named parameter doesn't exist.
Returns:
The string value of the configuration parameter.
"""
return getattr(_get_config(), name, default)
[docs]def get_all():
"""Return an object with an attribute for each conf parameter.
Returns:
An object with an attribute for each conf parameter.
"""
return _get_config()
