#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Wizard to generate bulkloader configuration.
Helper functions to call from the bulkloader.yaml.
The wizard is run by having bulkloader.py download datastore statistics
(https://developers.google.com/appengine/docs/python/datastore/stats,
specifically __Stat_PropertyType_PropertyName_Kind__) configured with
bulkloader_wizard.yaml.
"""
PROPERTY_DUPE_WARNING = (
' # Warning: This property is a duplicate, but with a different type.\n'
' # TODO: Edit this transform so only one property with this name '
'remains.\n')
KIND_PREAMBLE = """
- kind: %(kind_name)s
connector: # TODO: Choose a connector here: csv, simplexml, etc...
connector_options:
# TODO: Add connector options here--these are specific to each connector.
property_map:
- property: __key__
external_name: key
export_transform: transform.key_id_or_name_as_string
"""
[docs]class StatPostTransform(object):
"""Create text to insert between properties and filter out 'bad' properties.
This class is a callable post_export_function which saves state
across multiple calls.
It uses this saved state to determine if each entity is the first entity seen
of a new kind, a duplicate kind/propertyname entry, or just a new property
in the current kind being processed.
It will suppress bad output by returning None for NULL property types and
__private__ types (notably the stats themselves).
"""
def __init__(self):
"""Constructor.
Attributes:
seen_properties: (kind, propertyname) -> number of times seen before. If
seen more than once, this is a duplicate property for the kind.
last_seen: Previous kind seen. If it changes, this is a new kind.
"""
self.seen_properties = {}
self.last_seen = None
def __call__(self, instance, dictionary, bulkload_state):
"""Implementation of StatPropertyTypePropertyNameKindPostExport.
See class docstring for more info.
Args:
instance: Input, current entity being exported.
dictionary: Output, dictionary created by property_map transforms.
bulkload_state: Passed bulkload_state.
Returns:
Dictionary--same object as passed in dictionary.
"""
kind_name = dictionary['kind_name']
property_name = dictionary['property_name']
property_type = dictionary['property_type']
if kind_name.startswith('__'):
return None
if property_type == 'NULL':
return None
property_key = kind_name, property_name
if kind_name != self.last_seen:
self.last_seen = kind_name
separator = KIND_PREAMBLE % dictionary
elif property_key in self.seen_properties:
separator = PROPERTY_DUPE_WARNING % dictionary
else:
separator = ''
self.seen_properties[property_key] = (
self.seen_properties.get(property_key, 0) + 1)
dictionary['separator'] = separator
return dictionary
TYPE_TO_TRANSFORM_MAP = {
'Blob': ('transform.blobproperty_from_base64',
'base64.b64encode'),
'Boolean': ('transform.regexp_bool(\'true\', re.IGNORECASE)',
None),
'ByteString': ('transform.bytestring_from_base64', 'base64.b64encode'),
'Category': ('db.Category', None),
'Date/Time': ('transform.import_date_time(\'%Y-%m-%dT%H:%M:%S\')',
'transform.export_date_time(\'%Y-%m-%dT%H:%M:%S\')'),
'Email': ('db.Email', None),
'Float': ('transform.none_if_empty(float)', None),
'Integer': ('transform.none_if_empty(int)', None),
'Key': ('transform.create_foreign_key(\'TODO: fill in Kind name\')',
'transform.key_id_or_name_as_string'),
'Link': ('db.Link', None),
'PhoneNumber': ('db.PhoneNumber', None),
'PostalAddress': ('db.PostalAddress', None),
'Rating': ('transform.none_if_empty(db.Rating)', None),
'String': (None, None),
'Text': ('db.Text', None),
'User': ('transform.none_if_empty(users.User) # Assumes email address',
None),
}
