Stay organized with collections
Save and categorize content based on your preferences.
Source code for google.appengine.ext.bulkload.bulkloader_wizard
#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Wizard to generate bulkloader configuration.
Helper functions to call from the bulkloader.yaml.
The wizard is run by having bulkloader.py download datastore statistics
(https://developers.google.com/appengine/docs/python/datastore/stats,
specifically __Stat_PropertyType_PropertyName_Kind__) configured with
bulkloader_wizard.yaml.
"""
PROPERTY_DUPE_WARNING = (
' # Warning: This property is a duplicate, but with a different type.\n'
' # TODO: Edit this transform so only one property with this name '
'remains.\n')
KIND_PREAMBLE = """
- kind: %(kind_name)s
connector: # TODO: Choose a connector here: csv, simplexml, etc...
connector_options:
# TODO: Add connector options here--these are specific to each connector.
property_map:
- property: __key__
external_name: key
export_transform: transform.key_id_or_name_as_string
"""
[docs]class StatPostTransform(object):
"""Create text to insert between properties and filter out 'bad' properties.
This class is a callable post_export_function which saves state
across multiple calls.
It uses this saved state to determine if each entity is the first entity seen
of a new kind, a duplicate kind/propertyname entry, or just a new property
in the current kind being processed.
It will suppress bad output by returning None for NULL property types and
__private__ types (notably the stats themselves).
"""
def __init__(self):
"""Constructor.
Attributes:
seen_properties: (kind, propertyname) -> number of times seen before. If
seen more than once, this is a duplicate property for the kind.
last_seen: Previous kind seen. If it changes, this is a new kind.
"""
self.seen_properties = {}
self.last_seen = None
def __call__(self, instance, dictionary, bulkload_state):
"""Implementation of StatPropertyTypePropertyNameKindPostExport.
See class docstring for more info.
Args:
instance: Input, current entity being exported.
dictionary: Output, dictionary created by property_map transforms.
bulkload_state: Passed bulkload_state.
Returns:
Dictionary--same object as passed in dictionary.
"""
kind_name = dictionary['kind_name']
property_name = dictionary['property_name']
property_type = dictionary['property_type']
if kind_name.startswith('__'):
return None
if property_type == 'NULL':
return None
property_key = kind_name, property_name
if kind_name != self.last_seen:
self.last_seen = kind_name
separator = KIND_PREAMBLE % dictionary
elif property_key in self.seen_properties:
separator = PROPERTY_DUPE_WARNING % dictionary
else:
separator = ''
self.seen_properties[property_key] = (
self.seen_properties.get(property_key, 0) + 1)
dictionary['separator'] = separator
return dictionary
TYPE_TO_TRANSFORM_MAP = {
'Blob': ('transform.blobproperty_from_base64',
'base64.b64encode'),
'Boolean': ('transform.regexp_bool(\'true\', re.IGNORECASE)',
None),
'ByteString': ('transform.bytestring_from_base64', 'base64.b64encode'),
'Category': ('db.Category', None),
'Date/Time': ('transform.import_date_time(\'%Y-%m-%dT%H:%M:%S\')',
'transform.export_date_time(\'%Y-%m-%dT%H:%M:%S\')'),
'Email': ('db.Email', None),
'Float': ('transform.none_if_empty(float)', None),
'Integer': ('transform.none_if_empty(int)', None),
'Key': ('transform.create_foreign_key(\'TODO: fill in Kind name\')',
'transform.key_id_or_name_as_string'),
'Link': ('db.Link', None),
'PhoneNumber': ('db.PhoneNumber', None),
'PostalAddress': ('db.PostalAddress', None),
'Rating': ('transform.none_if_empty(db.Rating)', None),
'String': (None, None),
'Text': ('db.Text', None),
'User': ('transform.none_if_empty(users.User) # Assumes email address',
None),
}
Except as otherwise noted, the content of this page is licensed under the Creative Commons Attribution 4.0 License, and code samples are licensed under the Apache 2.0 License. For details, see the Google Developers Site Policies. Java is a registered trademark of Oracle and/or its affiliates.
Last updated 2025-06-16 UTC.
[[["Easy to understand","easyToUnderstand","thumb-up"],["Solved my problem","solvedMyProblem","thumb-up"],["Other","otherUp","thumb-up"]],[["Hard to understand","hardToUnderstand","thumb-down"],["Incorrect information or sample code","incorrectInformationOrSampleCode","thumb-down"],["Missing the information/samples I need","missingTheInformationSamplesINeed","thumb-down"],["Other","otherDown","thumb-down"]],["Last updated 2025-06-16 UTC."],[[["This code provides a wizard for generating bulkloader configuration files, aiding in the management of data import and export within Google App Engine."],["The `StatPostTransform` class manages state across multiple calls to identify the first entity of a new kind, duplicate properties, and suppress unwanted output from private or null property types."],["A `TYPE_TO_TRANSFORM_MAP` dictionary defines import and export transformations for various datastore types, which the `DatastoreTypeToTransforms` function uses to return the appropriate transforms for a given property type."],["The code handles duplicate properties of the same name with different types by adding a specific warning in the configuration file."],["The wizard is designed to work with the bulkloader.py tool, utilizing datastore statistics to configure the data loading and unloading process, with a template provided for each kind of data to define the needed configuration."]]],[]]