#
# Copyright 2008 The ndb Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MessageProperty -- a property storing ProtoRPC Message objects.
Basic usage:
Let's assume you have a protorpc.messages.Message subclass, like this:
from protorpc import messages
class Note(messages.Message):
text = messages.StringField(1, required=True)
when = messages.IntegerField(2)
Now suppose you'd like to store Notes in Cloud Datastore. Create a
model class to hold your notes, as follows:
from google.appengine.ext import ndb
from google.appengine.ext.ndb import msgprop
class NoteStore(ndb.Model):
note = msgprop.MessageProperty(Note)
name = ndb.StringProperty()
(The class name, 'NoteStore', and the property name, 'note', are yours
to choose.)
To store, a Note message, create a NoteStore entity and write it:
ns = NoteStore(note=my_note, name='foo')
key = ns.put()
To later retrieve the Note, read the NoteStore entity back:
ns = key.get()
my_note = ns.note
The MessageProperty class has many of the usual Property options:
- name: optional datastore name
- repeated: if True, stores a list of message values
- required: if True, the message value cannot be None
- default: optional default message value to store
- choices: optional list of allowed choices (must all be messages)
- validator: optional function to validate message values
- verbose_name: optional long name for the property
However, MessageProperty does not support the 'indexed' option.
Instead, you can specify a list of field names that will be indexed,
like this:
class MyStore(ndb.Model):
author = ndb.StringProperty()
note = msgprop.MessageProperty(Note, indexed_fields=['text', 'when'])
Now you can query for field values, like this:
stores = MyStore.query(MyStore.note.when >= 123).fetch()
Note the similarity with StructuredProperty -- in fact,
MessageProperty inherits from StructuredProperty. The main difference
is that StructuredProperty takes an Model subclass instead of a
protorpc.messages.Message subclass; and StructuredProperty doesn't
index any fields by default.
It works for nested messages (using MessageField) as well:
class Notes(messages.Message):
notes = messages.MessageField(Note, 1, repeated=True)
class MyNotesStore(ndb.Model):
author = ndb.StringProperty()
foo = msgprop.MessageProperty(Notes,
indexed_fields=['notes.text, 'notes.when'])
And given this value for indexed_fields, in this example you can also
query for subfields:
stores = MyNoteStore.query(MyNoteStore.foo.notes.when < 123).fetch()
A final option for MessageProperty is 'protocol'. This lets you
specify how the message object is serialized to Cloud Datastore. The
values are protocol names as used by protorpc.remote.Protocols class.
Supported protocol names are 'protobuf' and 'protojson'; the default
is 'protobuf'. (In the future this will use the global protocols
registry that is being added to protorpc; then any registered protocol
name will be acceptable.)
There is also an EnumProperty, which can be used to store a
messages.Enum value without wrapping it in a Message object. Example:
class Color(messages.Enum):
RED = 620
GREEN = 495
BLUE = 450
class Part(ndb.Model):
name = ndb.StringProperty()
color = msgprop.EnumProperty(Color, required=True)
p1 = Part(name='foo', color=Color.RED)
key1 = p1.put()
...
p2 = key1.get()
print p2.name, p2.color # prints "foo RED"
The EnumProperty stores the value as an integer; in fact, EnumProperty
is a subclass of IntegerProperty. This is handy to know, since it
implies that you can rename your enum values without having to modify
already-stored entities, but you cannot renumber them.
The EnumProperty supports the following standard options:
- name
- indexed
- repeated
- required
- default
- choices
- validator
- verbose_name
"""
from protorpc import messages
from protorpc import remote
from . import model
from . import utils
__all__ = ['MessageProperty', 'EnumProperty']
# TODO: Use ProtoRPC's global Protocols instance once it is in the SDK.
_protocols_registry = remote.Protocols.new_default()
_default_protocol = 'protobuf'
[docs]class EnumProperty(model.IntegerProperty):
"""Enums are represented in Cloud Datastore as integers.
While this is less user-friendly in the Datastore viewer, it matches
the representation of enums in the protobuf serialization (although
not in JSON), and it allows renaming enum values without requiring
changes to values already stored in the Datastore.
"""
_enum_type = None
# Insert enum_type as an initial positional argument.
_attributes = ['_enum_type'] + model.IntegerProperty._attributes
_positional = 1 + model.IntegerProperty._positional
@utils.positional(1 + _positional)
def __init__(self, enum_type, name=None, default=None, choices=None, **kwds):
"""Constructor.
Args:
enum_type: A subclass of protorpc.messages.Enum.
name: Optional datastore name (defaults to the property name).
Additional keywords arguments specify the same options as
supported by IntegerProperty.
"""
self._enum_type = enum_type
if default is not None:
self._validate(default)
if choices is not None:
map(self._validate, choices)
super(EnumProperty, self).__init__(name, default=default,
choices=choices, **kwds)
def _validate(self, value):
"""Validate an Enum value.
Raises:
TypeError if the value is not an instance of self._enum_type.
"""
if not isinstance(value, self._enum_type):
raise TypeError('Expected a %s instance, got %r instead' %
(self._enum_type.__name__, value))
def _to_base_type(self, enum):
"""Convert an Enum value to a base type (integer) value."""
return enum.number
def _from_base_type(self, val):
"""Convert a base type (integer) value to an Enum value."""
return self._enum_type(val)
def _analyze_indexed_fields(indexed_fields):
"""Internal helper to check a list of indexed fields.
Args:
indexed_fields: A list of names, possibly dotted names.
(A dotted name is a string containing names separated by dots,
e.g. 'foo.bar.baz'. An undotted name is a string containing no
dots, e.g. 'foo'.)
Returns:
A dict whose keys are undotted names. For each undotted name in
the argument, the dict contains that undotted name as a key with
None as a value. For each dotted name in the argument, the dict
contains the first component as a key with a list of remainders as
values.
Example:
If the argument is ['foo.bar.baz', 'bar', 'foo.bletch'], the return
value is {'foo': ['bar.baz', 'bletch'], 'bar': None}.
Raises:
TypeError if an argument is not a string.
ValueError for duplicate arguments and for conflicting arguments
(when an undotted name also appears as the first component of
a dotted name).
"""
result = {}
for field_name in indexed_fields:
if not isinstance(field_name, basestring):
raise TypeError('Field names must be strings; got %r' % (field_name,))
if '.' not in field_name:
if field_name in result:
raise ValueError('Duplicate field name %s' % field_name)
result[field_name] = None
else:
head, tail = field_name.split('.', 1)
if head not in result:
result[head] = [tail]
elif result[head] is None:
raise ValueError('Field name %s conflicts with ancestor %s' %
(field_name, head))
else:
result[head].append(tail)
return result
def _make_model_class(message_type, indexed_fields, **props):
"""Construct a Model subclass corresponding to a Message subclass.
Args:
message_type: A Message subclass.
indexed_fields: A list of dotted and undotted field names.
**props: Additional properties with which to seed the class.
Returns:
A Model subclass whose properties correspond to those fields of
message_type whose field name is listed in indexed_fields, plus
the properties specified by the **props arguments. For dotted
field names, a StructuredProperty is generated using a Model
subclass created by a recursive call.
Raises:
Whatever _analyze_indexed_fields() raises.
ValueError if a field name conflicts with a name in **props.
ValueError if a field name is not valid field of message_type.
ValueError if an undotted field name designates a MessageField.
"""
analyzed = _analyze_indexed_fields(indexed_fields)
for field_name, sub_fields in analyzed.iteritems():
if field_name in props:
raise ValueError('field name %s is reserved' % field_name)
try:
field = message_type.field_by_name(field_name)
except KeyError:
raise ValueError('Message type %s has no field named %s' %
(message_type.__name__, field_name))
if isinstance(field, messages.MessageField):
if not sub_fields:
raise ValueError(
'MessageField %s cannot be indexed, only sub-fields' % field_name)
sub_model_class = _make_model_class(field.type, sub_fields)
prop = model.StructuredProperty(sub_model_class, field_name,
repeated=field.repeated)
else:
if sub_fields is not None:
raise ValueError(
'Unstructured field %s cannot have indexed sub-fields' % field_name)
if isinstance(field, messages.EnumField):
prop = EnumProperty(field.type, field_name, repeated=field.repeated)
elif isinstance(field, messages.BytesField):
prop = model.BlobProperty(field_name,
repeated=field.repeated, indexed=True)
else:
# IntegerField, FloatField, BooleanField, StringField.
prop = model.GenericProperty(field_name, repeated=field.repeated)
props[field_name] = prop
return model.MetaModel('_%s__Model' % message_type.__name__,
(model.Model,), props)
[docs]class MessageProperty(model.StructuredProperty):
"""Messages are represented in Cloud Datastore as structured properties.
By default, the structured property has a single subproperty
containing the serialized message. This property is named 'blob_'
in Python but __<protocol>__ in the Datastore, where <protocol> is
the value of the protocol argument (default 'protobuf').
"""
_message_type = None
_indexed_fields = ()
_protocol = _default_protocol
_protocol_impl = None
# *Replace* first positional argument with _message_type, since the
# _modelclass attribute is synthetic.
_attributes = (['_message_type'] + model.StructuredProperty._attributes[1:] +
['_indexed_fields', '_protocol'])
@utils.positional(1 + model.StructuredProperty._positional)
def __init__(self, message_type, name=None,
indexed_fields=None, protocol=None, **kwds):
"""Constructor.
Args:
message_tyoe: A subclass of protorpc.messages.Message.
name: Optional datastore name (defaults to the property name).
indexed_fields: Optional list of dotted and undotted field names.
protocol: Optional protocol name default 'protobuf'.
Additional keywords arguments specify the same options as
supported by StructuredProperty, except 'indexed'.
"""
if not (isinstance(message_type, type) and
issubclass(message_type, messages.Message)):
raise TypeError('MessageProperty argument must be a Message subclass')
self._message_type = message_type
if indexed_fields is not None:
# TODO: Check they are all strings naming fields.
self._indexed_fields = tuple(indexed_fields)
# NOTE: Otherwise the class default i.e. (), prevails.
if protocol is None:
protocol = _default_protocol
self._protocol = protocol
self._protocol_impl = _protocols_registry.lookup_by_name(protocol)
blob_prop = model.BlobProperty('__%s__' % self._protocol)
# TODO: Solve this without reserving 'blob_'.
message_class = _make_model_class(message_type, self._indexed_fields,
blob_=blob_prop)
super(MessageProperty, self).__init__(message_class, name, **kwds)
def _validate(self, msg):
"""Validate an Enum value.
Raises:
TypeError if the value is not an instance of self._message_type.
"""
if not isinstance(msg, self._message_type):
raise TypeError('Expected a %s instance for %s property',
self._message_type.__name__,
self._code_name or self._name)
def _to_base_type(self, msg):
"""Convert a Message value to a Model instance (entity)."""
ent = _message_to_entity(msg, self._modelclass)
ent.blob_ = self._protocol_impl.encode_message(msg)
return ent
def _from_base_type(self, ent):
"""Convert a Model instance (entity) to a Message value."""
if ent._projection:
# Projection query result. Reconstitute the message from the fields.
return _projected_entity_to_message(ent, self._message_type)
blob = ent.blob_
if blob is not None:
protocol = self._protocol_impl
else:
# Perhaps it was written using a different protocol.
protocol = None
for name in _protocols_registry.names:
key = '__%s__' % name
if key in ent._values:
blob = ent._values[key]
if isinstance(blob, model._BaseValue):
blob = blob.b_val
protocol = _protocols_registry.lookup_by_name(name)
break
if blob is None or protocol is None:
return None # This will reveal the underlying dummy model.
msg = protocol.decode_message(self._message_type, blob)
return msg
def _message_to_entity(msg, modelclass):
"""Recursive helper for _to_base_type() to convert a message to an entity.
Args:
msg: A Message instance.
modelclass: A Model subclass.
Returns:
An instance of modelclass.
"""
ent = modelclass()
for prop_name, prop in modelclass._properties.iteritems():
if prop._code_name == 'blob_': # TODO: Devise a cleaner test.
continue # That's taken care of later.
value = getattr(msg, prop_name)
if value is not None and isinstance(prop, model.StructuredProperty):
if prop._repeated:
value = [_message_to_entity(v, prop._modelclass) for v in value]
else:
value = _message_to_entity(value, prop._modelclass)
setattr(ent, prop_name, value)
return ent
def _projected_entity_to_message(ent, message_type):
"""Recursive helper for _from_base_type() to convert an entity to a message.
Args:
ent: A Model instance.
message_type: A Message subclass.
Returns:
An instance of message_type.
"""
msg = message_type()
analyzed = _analyze_indexed_fields(ent._projection)
for name, sublist in analyzed.iteritems():
prop = ent._properties[name]
val = prop._get_value(ent)
assert isinstance(prop, model.StructuredProperty) == bool(sublist)
if sublist:
field = message_type.field_by_name(name)
assert isinstance(field, messages.MessageField)
assert prop._repeated == field.repeated
if prop._repeated:
assert isinstance(val, list)
val = [_projected_entity_to_message(v, field.type) for v in val]
else:
assert isinstance(val, prop._modelclass)
val = _projected_entity_to_message(val, field.type)
setattr(msg, name, val)
return msg
