Notice: Over the next few months, we're reorganizing the App Engine documentation site to make it easier to find content and better align with the rest of Google Cloud products. The same content will be available, but the navigation will now match the rest of the Cloud products. If you have feedback or questions as you navigate the site, click Send Feedback.

Python 2 is no longer supported by the community. We recommend that you migrate Python 2 apps to Python 3.
Stay organized with collections Save and categorize content based on your preferences.

Source code for google.appengine.api.search.search

#!/usr/bin/env python
#
# Copyright 2007 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""A Python Search API used by app developers.

Contains methods used to interface with Search API.
Contains API classes that forward to apiproxy.
"""








import base64
import datetime
import logging
import os
import re
import string
import sys
import warnings
from google.net.proto import ProtocolBuffer

if os.environ.get('APPENGINE_RUNTIME') == 'python27':
  from google.appengine.datastore import document_pb
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_types
  from google.appengine.api import namespace_manager
  from google.appengine.api.search import expression_parser
  from google.appengine.api.search import query_parser
  from google.appengine.api.search import search_service_pb
  from google.appengine.api.search import search_util
  from google.appengine.datastore import datastore_rpc
  from google.appengine.runtime import apiproxy_errors
else:
  from google.appengine.datastore import document_pb
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_types
  from google.appengine.api import namespace_manager
  from google.appengine.api.search import expression_parser
  from google.appengine.api.search import query_parser
  from google.appengine.api.search import search_service_pb
  from google.appengine.api.search import search_util
  from google.appengine.datastore import datastore_rpc
  from google.appengine.runtime import apiproxy_errors


__all__ = [
    'AtomField',
    'AtomFacet',
    'ConcurrentTransactionError',
    'Cursor',
    'DateField',
    'DeleteError',
    'DeleteResult',
    'Document',
    'DOCUMENT_ID_FIELD_NAME',
    'Error',
    'ExpressionError',
    'Facet',
    'FacetOptions',
    'FacetRange',
    'FacetRefinement',
    'FacetRequest',
    'FacetResult',
    'FacetResultValue',
    'Field',
    'FieldExpression',
    'HtmlField',
    'GeoField',
    'GeoPoint',
    'get_indexes',
    'get_indexes_async',
    'GetResponse',
    'Index',
    'InternalError',
    'InvalidRequest',
    'LANGUAGE_FIELD_NAME',
    'MatchScorer',
    'MAXIMUM_DOCUMENT_ID_LENGTH',
    'MAXIMUM_DOCUMENTS_PER_PUT_REQUEST',
    'MAXIMUM_DOCUMENTS_RETURNED_PER_SEARCH',
    'MAXIMUM_DEPTH_FOR_FACETED_SEARCH',
    'MAXIMUM_FACETS_TO_RETURN',
    'MAXIMUM_FACET_VALUES_TO_RETURN',
    'MAXIMUM_EXPRESSION_LENGTH',
    'MAXIMUM_FIELD_ATOM_LENGTH',
    'MAXIMUM_FIELD_NAME_LENGTH',
    'MAXIMUM_FIELD_PREFIX_LENGTH',
    'MAXIMUM_FIELD_VALUE_LENGTH',
    'MAXIMUM_FIELDS_RETURNED_PER_SEARCH',
    'MAXIMUM_GET_INDEXES_OFFSET',
    'MAXIMUM_INDEX_NAME_LENGTH',
    'MAXIMUM_INDEXES_RETURNED_PER_GET_REQUEST',
    'MAXIMUM_NUMBER_FOUND_ACCURACY',
    'MAXIMUM_QUERY_LENGTH',
    'MAXIMUM_SEARCH_OFFSET',
    'MAXIMUM_SORTED_DOCUMENTS',
    'MAX_DATE',
    'MAX_NUMBER_VALUE',
    'MIN_DATE',
    'MIN_NUMBER_VALUE',
    'NumberField',
    'NumberFacet',
    'OperationResult',
    'PutError',
    'PutResult',
    'Query',
    'QueryError',
    'QueryOptions',
    'RANK_FIELD_NAME',
    'RescoringMatchScorer',
    'SCORE_FIELD_NAME',
    'ScoredDocument',
    'SearchResults',
    'SortExpression',
    'SortOptions',
    'TextField',
    'Timeout',
    'TIMESTAMP_FIELD_NAME',
    'TokenizedPrefixField',
    'TransientError',
    'UntokenizedPrefixField',
    'VECTOR_FIELD_MAX_SIZE',
    'VectorField',
    ]

MAXIMUM_INDEX_NAME_LENGTH = 100
MAXIMUM_FIELD_VALUE_LENGTH = 1024 * 1024
MAXIMUM_FIELD_ATOM_LENGTH = 500
MAXIMUM_FIELD_PREFIX_LENGTH = 500
MAXIMUM_FIELD_NAME_LENGTH = 500
MAXIMUM_DOCUMENT_ID_LENGTH = 500
MAXIMUM_DOCUMENTS_PER_PUT_REQUEST = 200
MAXIMUM_EXPRESSION_LENGTH = 5000
MAXIMUM_QUERY_LENGTH = 2000
MAXIMUM_DOCUMENTS_RETURNED_PER_SEARCH = 1000
MAXIMUM_DEPTH_FOR_FACETED_SEARCH = 10000
MAXIMUM_FACETS_TO_RETURN = 100
MAXIMUM_FACET_VALUES_TO_RETURN = 100
MAXIMUM_SEARCH_OFFSET = 1000
MAXIMUM_SORTED_DOCUMENTS = 10000
MAXIMUM_NUMBER_FOUND_ACCURACY = 25000
MAXIMUM_FIELDS_RETURNED_PER_SEARCH = 1000
MAXIMUM_INDEXES_RETURNED_PER_GET_REQUEST = 1000
MAXIMUM_GET_INDEXES_OFFSET = 1000
VECTOR_FIELD_MAX_SIZE = 10000


DOCUMENT_ID_FIELD_NAME = '_doc_id'

LANGUAGE_FIELD_NAME = '_lang'

RANK_FIELD_NAME = '_rank'

SCORE_FIELD_NAME = '_score'



TIMESTAMP_FIELD_NAME = '_timestamp'





_LANGUAGE_RE = re.compile('^(.{2,3}|.{2}_.{2})$')

_MAXIMUM_STRING_LENGTH = 500
_MAXIMUM_CURSOR_LENGTH = 10000

_VISIBLE_PRINTABLE_ASCII = frozenset(
    set(string.printable) - set(string.whitespace))
_FIELD_NAME_PATTERN = '^[A-Za-z][A-Za-z0-9_]*$'

MAX_DATE = datetime.datetime(
    datetime.MAXYEAR, 12, 31, 23, 59, 59, 999999, tzinfo=None)
MIN_DATE = datetime.datetime(
    datetime.MINYEAR, 1, 1, 0, 0, 0, 0, tzinfo=None)


MAX_NUMBER_VALUE = 2147483647
MIN_NUMBER_VALUE = -2147483647


_PROTO_FIELDS_STRING_VALUE = frozenset(
    [document_pb.FieldValue.TEXT,
     document_pb.FieldValue.HTML,
     document_pb.FieldValue.ATOM,
     document_pb.FieldValue.UNTOKENIZED_PREFIX,
     document_pb.FieldValue.TOKENIZED_PREFIX])


[docs]class Error(Exception): """Indicates a call on the search API has failed."""
[docs]class InternalError(Error): """Indicates a call on the search API has failed on the internal backend."""
[docs]class TransientError(Error): """Indicates a call on the search API has failed, but retrying may succeed."""
[docs]class InvalidRequest(Error): """Indicates an invalid request was made on the search API by the client."""
[docs]class QueryError(Error): """An error occurred while parsing a query input string."""
[docs]class ExpressionError(Error): """An error occurred while parsing an expression input string."""
[docs]class Timeout(Error): """Indicates a call on the search API could not finish before its deadline."""
[docs]class ConcurrentTransactionError(Error): """Indicates a call on the search API failed due to concurrent updates."""
def _ConvertToUnicode(some_string): """Convert UTF-8 encoded string to unicode.""" if some_string is None: return None if isinstance(some_string, unicode): return some_string return unicode(some_string, 'utf-8') def _ConcatenateErrorMessages(prefix, status): """Returns an error message combining prefix and status.error_detail().""" if status.error_detail(): return prefix + ': ' + status.error_detail() return prefix class _RpcOperationFuture(object): """Represents the future result a search RPC sent to a backend.""" def __init__(self, call, request, response, deadline, get_result_hook): """Initializer. Args: call: Method name to call, as a string request: The request object response: The response object deadline: Deadline for RPC call in seconds; if None use the default. get_result_hook: Required result hook. Must be a function that takes no arguments. Its return value is returned by get_result(). """ _ValidateDeadline(deadline) self._get_result_hook = get_result_hook self._rpc = apiproxy_stub_map.UserRPC('search', deadline=deadline) self._rpc.make_call(call, request, response) def get_result(self): self._rpc.wait() try: self._rpc.check_success() except apiproxy_errors.ApplicationError, e: raise _ToSearchError(e) return self._get_result_hook() class _PutOperationFuture(_RpcOperationFuture): """Future specialized for Index put operations.""" def __init__(self, index, request, response, deadline, get_result_hook): super(_PutOperationFuture, self).__init__('IndexDocument', request, response, deadline, get_result_hook) self._index = index def get_result(self): try: return super(_PutOperationFuture, self).get_result() except apiproxy_errors.OverQuotaError, e: message = e.message + '; index = ' + self._index.name if self._index.namespace: message = message + ' in namespace ' + self._index.namespace raise apiproxy_errors.OverQuotaError(message) class _SimpleOperationFuture(object): """Adapts a late-binding function to a future.""" def __init__(self, future, function): self._future = future self._function = function def get_result(self): return self._function(self._future.get_result()) class _WrappedValueFuture(object): """Adapts an immediately-known result to a future.""" def __init__(self, result): self._result = result def get_result(self): return self._result def _ConvertToUTF8(value): if isinstance(value, float): value = repr(value) value = {'inf': 'Infinity', '-inf': '-Infinity', 'nan': 'NaN'}.get(value, value) elif isinstance(value, (int, long)): value = str(value) return _ConvertToUnicode(value).encode('utf-8')
[docs]class OperationResult(object): """Represents result of individual operation of a batch index or removal. This is an abstract class. """ (OK, INVALID_REQUEST, TRANSIENT_ERROR, INTERNAL_ERROR, TIMEOUT, CONCURRENT_TRANSACTION) = ( 'OK', 'INVALID_REQUEST', 'TRANSIENT_ERROR', 'INTERNAL_ERROR', 'TIMEOUT', 'CONCURRENT_TRANSACTION') _CODES = frozenset([OK, INVALID_REQUEST, TRANSIENT_ERROR, INTERNAL_ERROR, TIMEOUT, CONCURRENT_TRANSACTION]) def __init__(self, code, message=None, id=None): """Initializer. Args: code: The error or success code of the operation. message: An error message associated with any error. id: The id of the object some operation was performed on. Raises: TypeError: If an unknown attribute is passed. ValueError: If an unknown code is passed. """ self._message = _ConvertToUnicode(message) self._code = code if self._code not in self._CODES: raise ValueError('Unknown operation result code %r, must be one of %s' % (self._code, self._CODES)) self._id = _ConvertToUnicode(id) @property def code(self): """Returns the code indicating the status of the operation.""" return self._code @property def message(self): """Returns any associated error message if the operation was in error.""" return self._message @property def id(self): """Returns the Id of the object the operation was performed on.""" return self._id def __repr__(self): return _Repr(self, [('code', self.code), ('message', self.message), ('id', self.id)])
_ERROR_OPERATION_CODE_MAP = { search_service_pb.SearchServiceError.OK: OperationResult.OK, search_service_pb.SearchServiceError.INVALID_REQUEST: OperationResult.INVALID_REQUEST, search_service_pb.SearchServiceError.TRANSIENT_ERROR: OperationResult.TRANSIENT_ERROR, search_service_pb.SearchServiceError.INTERNAL_ERROR: OperationResult.INTERNAL_ERROR, search_service_pb.SearchServiceError.TIMEOUT: OperationResult.TIMEOUT, search_service_pb.SearchServiceError.CONCURRENT_TRANSACTION: OperationResult.CONCURRENT_TRANSACTION, }
[docs]class PutResult(OperationResult): """The result of indexing a single object."""
[docs]class DeleteResult(OperationResult): """The result of deleting a single document."""
[docs]class PutError(Error): """Indicates some error occurred indexing one of the objects requested.""" def __init__(self, message, results): """Initializer. Args: message: A message detailing the cause of the failure to index some document. results: A list of PutResult corresponding to the list of objects requested to be indexed. """ super(PutError, self).__init__(message) self._results = results @property def results(self): """Returns PutResult list corresponding to objects indexed.""" return self._results
[docs]class DeleteError(Error): """Indicates some error occured deleting one of the objects requested.""" def __init__(self, message, results): """Initializer. Args: message: A message detailing the cause of the failure to delete some document. results: A list of DeleteResult corresponding to the list of Ids of objects requested to be deleted. """ super(DeleteError, self).__init__(message) self._results = results @property def results(self): """Returns DeleteResult list corresponding to Documents deleted.""" return self._results
_ERROR_MAP = { search_service_pb.SearchServiceError.INVALID_REQUEST: InvalidRequest, search_service_pb.SearchServiceError.TRANSIENT_ERROR: TransientError, search_service_pb.SearchServiceError.INTERNAL_ERROR: InternalError, search_service_pb.SearchServiceError.TIMEOUT: Timeout, search_service_pb.SearchServiceError.CONCURRENT_TRANSACTION: ConcurrentTransactionError, } def _ToSearchError(error): """Translate an application error to a search Error, if possible. Args: error: An ApplicationError to translate. Returns: An Error if the error is known, otherwise the given apiproxy_errors.ApplicationError. """ if error.application_error in _ERROR_MAP: return _ERROR_MAP[error.application_error](error.error_detail) return error def _CheckInteger(value, name, zero_ok=True, upper_bound=None): """Checks whether value is an integer between the lower and upper bounds. Args: value: The value to check. name: The name of the value, to use in error messages. zero_ok: True if zero is allowed. upper_bound: The upper (inclusive) bound of the value. Optional. Returns: The checked value. Raises: ValueError: If the value is not a int or long, or is out of range. """ datastore_types.ValidateInteger(value, name, ValueError, empty_ok=True, zero_ok=zero_ok) if upper_bound is not None and value > upper_bound: raise ValueError('%s, %d must be <= %d' % (name, value, upper_bound)) return value def _CheckEnum(value, name, values=None): """Checks whether value is a member of the set of values given. Args: value: The value to check. name: The name of the value, to use in error messages. values: The iterable of possible values. Returns: The checked value. Raises: ValueError: If the value is not one of the allowable values. """ if value not in values: raise ValueError('%s, %r must be in %s' % (name, value, values)) return value def _IsFinite(value): """Returns whether a value is a finite number. Args: value: The value to check. Returns: True if the value is a finite number; otherwise False. """ if isinstance(value, float) and -1e30000 < value < 1e30000: return True elif isinstance(value, (int, long)): return True else: return False def _CheckNumber(value, name, should_be_finite=False): """Checks whether number value is of valid type and (optionally) finite. Args: value: The value to check. name: The name of the value, to use in error messages. should_be_finite: make sure the value is a finite number. Returns: The checked value. Raises: TypeError: If the value is not a number. ValueError: If should_be_finite is set and the value is not finite. """ if not isinstance(value, (int, long, float)): raise TypeError('%s must be a int, long or float, got %s' % (name, value.__class__.__name__)) if should_be_finite and not _IsFinite(value): raise ValueError('%s must be a finite value (got %f)' % (name, value)) return value def _CheckVector(value): """Checks whether vector value is of valid type and size. Args: value: the value to check. Returns: The checked value. Raises: TypeError: if any of vector elements are not a number. ValueError: if the size of the vector is greater than VECTOR_FIELD_MAX_SIZE or any of vector elements are not finite. """ if value is None: return if len(value) > VECTOR_FIELD_MAX_SIZE: raise ValueError('vector size must be less than %d' % VECTOR_FIELD_MAX_SIZE) for d in value: _CheckNumber(d, 'vector value', True) return value def _CheckStatus(status): """Checks whether a RequestStatus has a value of OK. Args: status: The RequestStatus to check. Raises: Error: A subclass of Error if the value of status is not OK. The subclass of Error is chosen based on value of the status code. InternalError: If the status value is unknown. """ if status.code() != search_service_pb.SearchServiceError.OK: if status.code() in _ERROR_MAP: raise _ERROR_MAP[status.code()](status.error_detail()) else: raise InternalError(status.error_detail()) def _ValidateString(value, name='unused', max_len=_MAXIMUM_STRING_LENGTH, empty_ok=False, type_exception=TypeError, value_exception=ValueError): """Raises an exception if value is not a valid string or a subclass thereof. A string is valid if it's not empty, no more than _MAXIMUM_STRING_LENGTH bytes. The exception type can be specified with the exception arguments for type and value issues. Args: value: The value to validate. name: The name of this value; used in the exception message. max_len: The maximum allowed length, in bytes. empty_ok: Allow empty value. type_exception: The type of exception to raise if not a basestring. value_exception: The type of exception to raise if invalid value. Returns: The checked string. Raises: TypeError: If value is not a basestring or subclass. ValueError: If the value is None or longer than max_len. """ if value is None and empty_ok: return if value is not None and not isinstance(value, basestring): raise type_exception('%s must be a basestring; got %s:' % (name, value.__class__.__name__)) if not value and not empty_ok: raise value_exception('%s must not be empty.' % name) if len(value.encode('utf-8')) > max_len: raise value_exception('%s must be under %d bytes.' % (name, max_len)) return value def _ValidateVisiblePrintableAsciiNotReserved(value, name): """Checks if value is a visible printable ASCII string not starting with '!'. Whitespace characters are excluded. Printable visible ASCII strings starting with '!' are reserved for internal use. Args: value: The string to validate. name: The name of this string; used in the exception message. Returns: The checked string. Raises: ValueError: If the string is not visible printable ASCII, or starts with '!'. """ for char in value: if char not in _VISIBLE_PRINTABLE_ASCII: raise ValueError( '%r must be visible printable ASCII: %r' % (name, value)) if value.startswith('!'): raise ValueError('%r must not start with "!": %r' % (name, value)) return value def _CheckIndexName(index_name): """Checks index_name is a string which is not too long, and returns it. Index names must be visible printable ASCII and not start with '!'. """ _ValidateString(index_name, 'index name', MAXIMUM_INDEX_NAME_LENGTH) return _ValidateVisiblePrintableAsciiNotReserved(index_name, 'index_name') def _CheckFacetName(name): """Checks facet name is not too long and matches facet name pattern. Facet name pattern: "[A-Za-z][A-Za-z0-9_]*". Args: name: the name string to validate. Returns: the valid name. """ return _CheckFieldName(name) def _CheckFieldName(name): """Checks field name is not too long and matches field name pattern. Field name pattern: "[A-Za-z][A-Za-z0-9_]*". """ _ValidateString(name, 'name', MAXIMUM_FIELD_NAME_LENGTH) if not re.match(_FIELD_NAME_PATTERN, name): raise ValueError('field name "%s" should match pattern: %s' % (name, _FIELD_NAME_PATTERN)) return name def _CheckExpression(expression): """Checks whether the expression is a string.""" expression = _ValidateString(expression, max_len=MAXIMUM_EXPRESSION_LENGTH) try: expression_parser.Parse(expression) except expression_parser.ExpressionException, e: raise ExpressionError('Failed to parse expression "%s"' % expression) return expression def _CheckFieldNames(names): """Checks each name in names is a valid field name.""" for name in names: _CheckFieldName(name) return names def _GetList(a_list): """Utility function that converts None to the empty list.""" if a_list is None: return [] else: return list(a_list) def _ConvertToList(arg): """Converts arg to a list, empty if None, single element if not a list.""" if isinstance(arg, basestring): return [arg] if arg is not None: try: return list(iter(arg)) except TypeError: return [arg] return [] def _CheckType(obj, obj_type, obj_name): """Check the type of an object.""" if not isinstance(obj, obj_type): raise TypeError('%s must be a %s, got %s' % (obj_name, obj_type, obj.__class__.__name__)) return obj def _ConvertToListAndCheckType(arg, element_type, arg_name): """Converts args to a list and check its element type.""" ret = _ConvertToList(arg) for element in ret: if not isinstance(element, element_type): raise TypeError('%s should be single element or list of type %s' % (arg_name, element_type)) return ret def _ConvertToUnicodeList(arg): """Converts arg to a list of unicode objects.""" return [_ConvertToUnicode(value) for value in _ConvertToList(arg)] def _CheckDocumentId(doc_id): """Checks doc_id is a valid document identifier, and returns it. Document ids must be visible printable ASCII and not start with '!'. """ _ValidateString(doc_id, 'doc_id', MAXIMUM_DOCUMENT_ID_LENGTH) _ValidateVisiblePrintableAsciiNotReserved(doc_id, 'doc_id') return doc_id def _CheckText(value, name='value', empty_ok=True): """Checks the field text is a valid string.""" return _ValidateString(value, name, MAXIMUM_FIELD_VALUE_LENGTH, empty_ok) def _CheckHtml(html): """Checks the field html is a valid HTML string.""" return _ValidateString(html, 'html', MAXIMUM_FIELD_VALUE_LENGTH, empty_ok=True) def _CheckAtom(atom): """Checks the field atom is a valid string.""" return _ValidateString(atom, 'atom', MAXIMUM_FIELD_ATOM_LENGTH, empty_ok=True) def _CheckPrefix(prefix): """Checks if the untokenized or tokenized prefix field is a valid string.""" return _ValidateString(prefix, 'prefix', MAXIMUM_FIELD_PREFIX_LENGTH, empty_ok=True) def _CheckDate(date): """Checks the date is in the correct range.""" if isinstance(date, datetime.datetime): if date < MIN_DATE or date > MAX_DATE: raise TypeError('date must be between %s and %s (got %s)' % (MIN_DATE, MAX_DATE, date)) elif isinstance(date, datetime.date): if date < MIN_DATE.date() or date > MAX_DATE.date(): raise TypeError('date must be between %s and %s (got %s)' % (MIN_DATE, MAX_DATE, date)) else: raise TypeError('date must be datetime.datetime or datetime.date') return date def _CheckLanguage(language): """Checks language is None or a string that matches _LANGUAGE_RE.""" if language is None: return None if not isinstance(language, basestring): raise TypeError('language must be a basestring, got %s' % language.__class__.__name__) if not re.match(_LANGUAGE_RE, language): raise ValueError('invalid language %s. Languages should be two letters.' % language) return language def _CheckDocument(document): """Check that the document is valid. This checks for all server-side requirements on Documents. Currently, that means ensuring that there are no repeated number, date, or vector fields. Args: document: The search.Document to check for validity. Raises: ValueError: if the document is invalid in a way that would trigger a PutError from the server. """ no_repeat_vector_names = set() no_repeat_date_names = set() no_repeat_number_names = set() for field in document.fields: if isinstance(field, NumberField): if field.name in no_repeat_number_names: raise ValueError( 'Invalid document %s: field %s with type date or number may not ' 'be repeated.' % (document.doc_id, field.name)) no_repeat_number_names.add(field.name) elif isinstance(field, DateField): if field.name in no_repeat_date_names: raise ValueError( 'Invalid document %s: field %s with type date or number may not ' 'be repeated.' % (document.doc_id, field.name)) no_repeat_date_names.add(field.name) elif isinstance(field, VectorField): if field.name in no_repeat_vector_names: raise ValueError( 'Invalid document %s: field %s with type vector may not ' 'be repeated.' % (document.doc_id, field.name)) no_repeat_vector_names.add(field.name) def _CheckSortLimit(limit): """Checks the limit on number of docs to score or sort is not too large.""" return _CheckInteger(limit, 'limit', upper_bound=MAXIMUM_SORTED_DOCUMENTS) def _Repr(class_instance, ordered_dictionary): """Generates an unambiguous representation for instance and ordered dict.""" return u'search.%s(%s)' % (class_instance.__class__.__name__, ', '.join( ['%s=%r' % (key, value) for (key, value) in ordered_dictionary if value is not None and value != []])) def _ListIndexesResponsePbToGetResponse(response, include_schema): """Returns a GetResponse constructed from get_indexes response pb.""" return GetResponse( results=[_NewIndexFromPb(index, include_schema) for index in response.index_metadata_list()])
[docs]@datastore_rpc._positional(7) def get_indexes(namespace='', offset=None, limit=20, start_index_name=None, include_start_index=True, index_name_prefix=None, fetch_schema=False, deadline=None, **kwargs): """Returns a list of available indexes. Args: namespace: The namespace of indexes to be returned. If not set then the current namespace is used. offset: The offset of the first returned index. limit: The number of indexes to return. start_index_name: The name of the first index to be returned. include_start_index: Whether or not to return the start index. index_name_prefix: The prefix used to select returned indexes. fetch_schema: Whether to retrieve Schema for each Index or not. Kwargs: deadline: Deadline for RPC call in seconds; if None use the default. Returns: The GetResponse containing a list of available indexes. Raises: InternalError: If the request fails on internal servers. TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values (e.g., a negative deadline). """ return get_indexes_async( namespace, offset, limit, start_index_name, include_start_index, index_name_prefix, fetch_schema, deadline=deadline, **kwargs).get_result()
[docs]@datastore_rpc._positional(7) def get_indexes_async(namespace='', offset=None, limit=20, start_index_name=None, include_start_index=True, index_name_prefix=None, fetch_schema=False, deadline=None, **kwargs): """Asynchronously returns a list of available indexes. Identical to get_indexes() except that it returns a future. Call get_result() on the return value to block on the call and get its result. """ app_id = kwargs.pop('app_id', None) if kwargs: raise TypeError('Invalid arguments: %s' % ', '.join(kwargs)) request = search_service_pb.ListIndexesRequest() params = request.mutable_params() if namespace is None: namespace = namespace_manager.get_namespace() if namespace is None: namespace = u'' namespace_manager.validate_namespace(namespace, exception=ValueError) params.set_namespace(namespace) if offset is not None: params.set_offset(_CheckInteger(offset, 'offset', zero_ok=True, upper_bound=MAXIMUM_GET_INDEXES_OFFSET)) params.set_limit(_CheckInteger( limit, 'limit', zero_ok=False, upper_bound=MAXIMUM_INDEXES_RETURNED_PER_GET_REQUEST)) if start_index_name is not None: params.set_start_index_name( _ValidateString(start_index_name, 'start_index_name', MAXIMUM_INDEX_NAME_LENGTH, empty_ok=False)) if include_start_index is not None: params.set_include_start_index(bool(include_start_index)) if index_name_prefix is not None: params.set_index_name_prefix( _ValidateString(index_name_prefix, 'index_name_prefix', MAXIMUM_INDEX_NAME_LENGTH, empty_ok=False)) params.set_fetch_schema(fetch_schema) response = search_service_pb.ListIndexesResponse() if app_id: request.set_app_id(app_id) def hook(): _CheckStatus(response.status()) return _ListIndexesResponsePbToGetResponse(response, fetch_schema) return _RpcOperationFuture( 'ListIndexes', request, response, deadline, hook)
[docs]class Field(object): """An abstract base class which represents a field of a document. This class should not be directly instantiated. """ (TEXT, HTML, ATOM, DATE, NUMBER, GEO_POINT, UNTOKENIZED_PREFIX, TOKENIZED_PREFIX, VECTOR) = ('TEXT', 'HTML', 'ATOM', 'DATE', 'NUMBER', 'GEO_POINT', 'UNTOKENIZED_PREFIX', 'TOKENIZED_PREFIX', 'VECTOR') _FIELD_TYPES = frozenset([TEXT, HTML, ATOM, DATE, NUMBER, GEO_POINT, UNTOKENIZED_PREFIX, TOKENIZED_PREFIX, VECTOR]) def __init__(self, name, value, language=None): """Initializer. Args: name: The name of the field. Field names must have maximum length MAXIMUM_FIELD_NAME_LENGTH and match pattern "[A-Za-z][A-Za-z0-9_]*". value: The value of the field which can be a str, unicode or date. language: The ISO 693-1 two letter code of the language used in the value. See http://www.sil.org/iso639-3/codes.asp?order=639_1&letter=%25 for a list of valid codes. Correct specification of language code will assist in correct tokenization of the field. If None is given, then the language code of the document will be used. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ self._name = _CheckFieldName(_ConvertToUnicode(name)) self._value = self._CheckValue(value) self._language = _CheckLanguage(_ConvertToUnicode(language)) @property def name(self): """Returns the name of the field.""" return self._name @property def language(self): """Returns the code of the language the content in value is written in.""" return self._language @property def value(self): """Returns the value of the field.""" return self._value def _CheckValue(self, value): """Checks the value is valid for the given type. Args: value: The value to check. Returns: The checked value. """ raise NotImplementedError('_CheckValue is an abstract method') def __repr__(self): return _Repr(self, [('name', self.name), ('language', self.language), ('value', self.value)]) def __eq__(self, other): return isinstance(other, type(self)) and self.__key() == other.__key() def __ne__(self, other): return not self == other def __key(self): return (self.name, self.value, self.language) def __hash__(self): return hash(self.__key()) def __str__(self): return repr(self) def _CopyStringValueToProtocolBuffer(self, field_value_pb): """Copies value to a string value in proto buf.""" field_value_pb.set_string_value(self.value.encode('utf-8'))
[docs]class Facet(object): """An abstract base class which represents a facet of a document. This class should not be directly instantiated. """ def __init__(self, name, value): """Initializer. Args: name: The name of the facet. Facet names must have maximum length MAXIMUM_FIELD_NAME_LENGTH and match pattern "[A-Za-z][A-Za-z0-9_]*". value: The value of the facet which can be a str, unicode or number. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ self._name = _CheckFacetName(_ConvertToUnicode(name)) self._value = self._CheckValue(value) @property def name(self): """Returns the name of the facet.""" return self._name @property def value(self): """Returns the value of the facet.""" return self._value @classmethod def _CheckValue(cls, value): """Checks the value is valid for the given type. Args: value: The value to check. Returns: The checked value. """ raise NotImplementedError('_CheckValue is an abstract method') def _CopyStringValueToProtocolBuffer(self, facet_value_pb): """Copies value to a string value in proto buf.""" facet_value_pb.set_string_value(self.value.encode('utf-8')) def _CopyToProtocolBuffer(self, pb): """Copies facet's contents to a document_pb.Facet proto buffer.""" pb.set_name(self.name) if self.value is not None: facet_value_pb = pb.mutable_value() self._CopyValueToProtocolBuffer(facet_value_pb) return pb def _AttributeValueList(self): return [self.name, self.value] def __eq__(self, other): return (isinstance(other, type(self)) and self._AttributeValueList() == other._AttributeValueList()) def __ne__(self, other): return not self == other def __hash__(self): return hash(self._AttributeValueList()) def __repr__(self): return _Repr(self, [('name', self.name), ('value', self.value)])
[docs]class AtomFacet(Facet): """A Facet that has content to be treated as a single token for indexing. The following example shows an atom facet named wine_type: AtomFacet(name='wine_type', value='Red') """ def __init__(self, name, value=None): """Initializer. Args: name: The name of the facet. value: A str or unicode object to be treated as an indivisible text value. Raises: TypeError: If value is not a string. ValueError: If value is longer than allowed. """ Facet.__init__(self, name, _ConvertToUnicode(value)) @classmethod def _CheckValue(cls, value): return _CheckAtom(value) def _CopyValueToProtocolBuffer(self, facet_value_pb): facet_value_pb.set_type(document_pb.FacetValue.ATOM) self._CopyStringValueToProtocolBuffer(facet_value_pb)
[docs]class NumberFacet(Facet): """A Facet that has a numeric value. The following example shows a number facet named wine_vintage: NumberFacet(name='wine_vintage', value=2000) """ def __init__(self, name, value=None): """Initializer. Args: name: The name of the facet. value: A numeric value. Raises: TypeError: If value is not numeric. ValueError: If value is out of range. """ Facet.__init__(self, name, value) @classmethod def _CheckValue(cls, value): _CheckNumber(value, 'number facet value', True) if value >= MIN_NUMBER_VALUE and value <= MAX_NUMBER_VALUE: return value raise ValueError('value must be between %f and %f (got %f)' % (MIN_NUMBER_VALUE, MAX_NUMBER_VALUE, value)) def _CopyValueToProtocolBuffer(self, facet_value_pb): facet_value_pb.set_type(document_pb.FacetValue.NUMBER) facet_value_pb.set_string_value(_ConvertToUTF8(self.value))
def _NewFacetFromPb(pb): """Constructs a Facet from a document_pb.Facet protocol buffer.""" name = _DecodeUTF8(pb.name()) val_type = pb.value().type() value = _DecodeValue(_GetFacetValue(pb.value()), val_type) if val_type == document_pb.FacetValue.ATOM: return AtomFacet(name, value) elif val_type == document_pb.FacetValue.NUMBER: return NumberFacet(name, value) return InvalidRequest('Unknown facet value type %d' % val_type) def _NewFacetsFromPb(facet_list): """Returns a list of Facet copied from a document_pb.Document proto buf.""" return [_NewFacetFromPb(f) for f in facet_list]
[docs]class FacetRange(object): """A facet range with start and end values. An example of a FacetRange for good rating is: FacetRange(start=3.0, end=3.5) """ @datastore_rpc._positional(1) def __init__(self, start=None, end=None): """Initializer. Args: start: Start value for the range, inclusive. end: End value for the range. exclusive. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ if start is None and end is None: raise ValueError( 'Either start or end need to be provided for a facet range.') none_or_numeric_type = (type(None), int, float, long) self._start = _CheckType(start, none_or_numeric_type, 'start') self._end = _CheckType(end, none_or_numeric_type, 'end') if self._start is not None: NumberFacet._CheckValue(self._start) if self._end is not None: NumberFacet._CheckValue(self._end) @property def start(self): """Returns inclusive start of the range.""" return self._start @property def end(self): """Returns exclusive end of the range.""" return self._end def __repr__(self): return _Repr(self, [('start', self.start), ('end', self.end)]) def _CopyToProtocolBuffer(self, range_pb): if self.start is not None: range_pb.set_start(_ConvertToUTF8(self.start)) if self.end is not None: range_pb.set_end(_ConvertToUTF8(self.end))
[docs]class FacetRequest(object): """A facet to be included in search result. An example of a request for a facet only with name: FacetRequest('ExpediteShipping') (in that case, results will always have this facet) Or with a value constraint: FacetRequest('Size', values=['XL','L','M'] (results will have this facet with only specified values) Or ranges: FacetRequest('Rating', ranges=[ FacetRange(1.0, 2.0), FacetRange(2.0, 3.5), FacetRange(3.5, 4.0)] (results will have this facet with specified ranges) """ @datastore_rpc._positional(2) def __init__(self, name, value_limit=10, ranges=None, values=None): """Initializer. Args: name: The name of the facet. value_limit: Number of values to return if values is not specified. ranges: Range of values to return. Cannot be set with values. values: Specific values to return. Cannot be set with ranges. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ self._name = _CheckFacetName(_ConvertToUnicode(name)) self._value_limit = _CheckFacetValueLimit(value_limit) if ranges is not None and values is not None: raise ValueError( 'Cannot specify both ranges and values.') self._ranges = _ConvertToListAndCheckType( ranges, FacetRange, 'ranges') self._values = _ConvertToListAndCheckType( values, (basestring, int, float, long), 'values') for value in self._values: if isinstance(value, (int, float, long)): NumberFacet._CheckValue(value) @property def name(self): """Returns the name of the facet.""" return self._name @property def value_limit(self): """Returns number of values to be included in the result.""" return self._value_limit @property def ranges(self): """Returns FacetRanges of values to be included in the result.""" return self._ranges @property def values(self): """Returns specific values to be included in the result.""" return self._values def _CopyToProtocolBuffer(self, facet_request_pb): """Converts this object to a search_service_pb.FacetRequest proto buff.""" facet_request_pb.set_name(self.name) request_param_pb = facet_request_pb.mutable_params() request_param_pb.set_value_limit(self.value_limit) for facet_range in self.ranges: facet_range._CopyToProtocolBuffer(request_param_pb.add_range()) for constraint in self.values: request_param_pb.add_value_constraint(_ConvertToUTF8(constraint)) def __repr__(self): return _Repr(self, [('name', self.name), ('value_limit', self.value_limit), ('ranges', self.ranges), ('values', self.values)])
[docs]class FacetRefinement(object): """A Facet Refinement to filter out search results based on a facet value. NOTE: The recommended way to use facet refinement is to use the token string. Each FacetResult will have a token that is acceptable instead of this class. To provide manual FacetRefinement, an instance of this class can be passed to SearchOptions. NOTE: that either value or facet_range should be set but not both. Example: Request for a range refinement for a numeric facet: FacetRefinement(name='rating', facet_range=FacetRange(start=1.0,end=2.5)) """ @datastore_rpc._positional(2) def __init__(self, name, value=None, facet_range=None): """Initializer. Args: name: The name of the facet. value: Value of the facet. facet_range: A FacetRange to refine facet based on a range. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ self._name = _ConvertToUnicode(name) if (value is None) == (facet_range is None): raise ValueError('Either value or facet_range should be set but not ' 'both.') self._value = value self._facet_range = facet_range @property def name(self): """Returns name of the facet refinement.""" return self._name @property def value(self): """Returns value of the facet refinement.""" return self._value @property def facet_range(self): """Returns range of the facet refinement.""" return self._facet_range
[docs] def ToTokenString(self): """Converts this refinement to a token string safe to be used in HTML. The format of this string may change. Returns: A token string safe to be used in HTML for this facet refinement. """ facet_refinement = search_service_pb.FacetRefinement() self._CopyToProtocolBuffer(facet_refinement) return base64.b64encode(facet_refinement.SerializeToString())
[docs] @staticmethod def FromTokenString(token_string): """Converts a token string to a FacetRefinement object. Do not store token strings between different versions of API as key could be incompatible. Args: token_string: A token string created by ToTokenString method or returned by a search result. Returns: A FacetRefinement object. Raises: ValueError: If the token_string is invalid. """ ref_pb = search_service_pb.FacetRefinement() try: ref_pb.ParseFromString(base64.b64decode(token_string)) except (ProtocolBuffer.ProtocolBufferDecodeError, TypeError), e: raise ValueError('Invalid refinement token %s' % token_string, e) facet_range = None if ref_pb.has_range(): range_pb = ref_pb.range() facet_range = FacetRange( start=float(range_pb.start()) if range_pb.has_start() else None, end=float(range_pb.end()) if range_pb.has_end() else None) return FacetRefinement(ref_pb.name(), value=ref_pb.value() if ref_pb.has_value() else None, facet_range=facet_range)
def _CopyToProtocolBuffer(self, facet_refinement_pb): """Copies This object to a search_service_pb.FacetRefinement.""" facet_refinement_pb.set_name(self.name) if self.value is not None: facet_refinement_pb.set_value(_ConvertToUTF8(self.value)) if self.facet_range is not None: self.facet_range._CopyToProtocolBuffer( facet_refinement_pb.mutable_range()) def __repr__(self): return _Repr(self, [('name', self.name), ('value', self.value), ('facet_range', self.facet_range)])
def _CopyFieldToProtocolBuffer(field, pb): """Copies field's contents to a document_pb.Field protocol buffer.""" pb.set_name(field.name.encode('utf-8')) field_value_pb = pb.mutable_value() if field.language: field_value_pb.set_language(field.language.encode('utf-8')) if field.value is not None: field._CopyValueToProtocolBuffer(field_value_pb) return pb
[docs]class TextField(Field): """A Field that has text content. The following example shows a text field named signature with Polish content: TextField(name='signature', value='brzydka pogoda', language='pl') """ def __init__(self, name, value=None, language=None): """Initializer. Args: name: The name of the field. value: A str or unicode object containing text. language: The code of the language the value is encoded in. Raises: TypeError: If value is not a string. ValueError: If value is longer than allowed. """ Field.__init__(self, name, _ConvertToUnicode(value), language) def _CheckValue(self, value): return _CheckText(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.TEXT) self._CopyStringValueToProtocolBuffer(field_value_pb)
[docs]class HtmlField(Field): """A Field that has HTML content. The following example shows an html field named content: HtmlField(name='content', value='<html>herbata, kawa</html>', language='pl') """ def __init__(self, name, value=None, language=None): """Initializer. Args: name: The name of the field. value: A str or unicode object containing the searchable content of the Field. language: The code of the language the value is encoded in. Raises: TypeError: If value is not a string. ValueError: If value is longer than allowed. """ Field.__init__(self, name, _ConvertToUnicode(value), language) def _CheckValue(self, value): return _CheckHtml(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.HTML) self._CopyStringValueToProtocolBuffer(field_value_pb)
[docs]class AtomField(Field): """A Field that has content to be treated as a single token for indexing. The following example shows an atom field named contributor: AtomField(name='contributor', value='foo@bar.com') """ def __init__(self, name, value=None, language=None): """Initializer. Args: name: The name of the field. value: A str or unicode object to be treated as an indivisible text value. language: The code of the language the value is encoded in. Raises: TypeError: If value is not a string. ValueError: If value is longer than allowed. """ Field.__init__(self, name, _ConvertToUnicode(value), language) def _CheckValue(self, value): return _CheckAtom(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.ATOM) self._CopyStringValueToProtocolBuffer(field_value_pb)
[docs]class VectorField(Field): """A vector field that can be used in a dot product expression. The following example shows a vector field named scores: VectorField(name='scores', value=[1, 2, 3]) That can be used in a sort/field expression like this: dot(scores, vector(3, 2, 1)) """ def __init__(self, name, value=None): """Initializer. Args: name: The name of the field. value: The vector field value. Raises: TypeError: If vector elements are not numbers. ValueError: If value elements are not finite numbers. """ Field.__init__(self, name, _GetList(value)) def _CheckValue(self, value): return _CheckVector(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.VECTOR) for d in self.value: field_value_pb.add_vector_value(d)
[docs]class UntokenizedPrefixField(Field): """A field that matches searches on prefixes of the whole field. The following example shows an untokenized prefix field named title: UntokenizedPrefixField(name='title', value='how to swim freestyle') """ def __init__(self, name, value=None, language=None): """Initializer. Args: name: The name of the field. value: The untokenized prefix field value. language: The code of the language the value is encoded in. Raises: TypeError: If value is not a string. ValueError: If value is longer than allowed. """ Field.__init__(self, name, _ConvertToUnicode(value), language) def _CheckValue(self, value): return _CheckPrefix(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.UNTOKENIZED_PREFIX) self._CopyStringValueToProtocolBuffer(field_value_pb)
[docs]class TokenizedPrefixField(Field): """A field that matches searches on prefixes of its individual terms. The following example shows a tokenized prefix field named title: TokenizedPrefixField(name='title', value='Goodwill Hunting') """ def __init__(self, name, value=None, language=None): """Initializer. Args: name: The name of the field. value: The tokenized prefix field value. language: The code of the language the value is encoded in. Raises: TypeError: If value is not a string. ValueError: If value is longer than allowed. """ Field.__init__(self, name, _ConvertToUnicode(value), language) def _CheckValue(self, value): return _CheckPrefix(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.TOKENIZED_PREFIX) self._CopyStringValueToProtocolBuffer(field_value_pb)
[docs]class DateField(Field): """A Field that has a date or datetime value. Only Python "naive" date or datetime values may be used (not "aware" values). The following example shows a date field named creation_date: DateField(name='creation_date', value=datetime.date(2011, 03, 11)) """ def __init__(self, name, value=None): """Initializer. Args: name: The name of the field. value: A datetime.date or a datetime.datetime. Raises: TypeError: If value is not a datetime.date or a datetime.datetime. """ Field.__init__(self, name, value) def _CheckValue(self, value): return _CheckDate(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.DATE) field_value_pb.set_string_value(search_util.SerializeDate(self.value))
[docs]class NumberField(Field): """A Field that has a numeric value. The following example shows a number field named size: NumberField(name='size', value=10) """ def __init__(self, name, value=None): """Initializer. Args: name: The name of the field. value: A numeric value. Raises: TypeError: If value is not numeric. ValueError: If value is out of range. """ Field.__init__(self, name, value) def _CheckValue(self, value): value = _CheckNumber(value, 'field value', True) if value is not None and (value < MIN_NUMBER_VALUE or value > MAX_NUMBER_VALUE): raise ValueError('value, %d must be between %d and %d' % (value, MIN_NUMBER_VALUE, MAX_NUMBER_VALUE)) return value def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.NUMBER) field_value_pb.set_string_value(str(self.value))
[docs]class GeoPoint(object): """Represents a point on the Earth's surface, in lat, long coordinates.""" def __init__(self, latitude, longitude): """Initializer. Args: latitude: The angle between the equatorial plan and a line that passes through the GeoPoint, between -90 and 90 degrees. longitude: The angle east or west from a reference meridian to another meridian that passes through the GeoPoint, between -180 and 180 degrees. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ self._latitude = self._CheckLatitude(latitude) self._longitude = self._CheckLongitude(longitude) @property def latitude(self): """Returns the angle between equatorial plan and line thru the geo point.""" return self._latitude @property def longitude(self): """Returns the angle from a reference meridian to another meridian.""" return self._longitude def _CheckLatitude(self, value): _CheckNumber(value, 'latitude', True) if value < -90.0 or value > 90.0: raise ValueError('latitude must be between -90 and 90 degrees ' 'inclusive, was %f' % value) return value def _CheckLongitude(self, value): _CheckNumber(value, 'longitude', True) if value < -180.0 or value > 180.0: raise ValueError('longitude must be between -180 and 180 degrees ' 'inclusive, was %f' % value) return value def __eq__(self, other): return (self.latitude == other.latitude and self.longitude == other.longitude) def __repr__(self): return _Repr(self, [('latitude', self.latitude), ('longitude', self.longitude)])
def _CheckGeoPoint(geo_point): """Checks geo_point is a GeoPoint and returns it.""" if not isinstance(geo_point, GeoPoint): raise TypeError('geo_point must be a GeoPoint, got %s' % geo_point.__class__.__name__) return geo_point
[docs]class GeoField(Field): """A Field that has a GeoPoint value. The following example shows a geo field named place: GeoField(name='place', value=GeoPoint(latitude=-33.84, longitude=151.26)) """ def __init__(self, name, value=None): """Initializer. Args: name: The name of the field. value: A GeoPoint value. Raises: TypeError: If value is not numeric. """ Field.__init__(self, name, value) def _CheckValue(self, value): return _CheckGeoPoint(value) def _CopyValueToProtocolBuffer(self, field_value_pb): field_value_pb.set_type(document_pb.FieldValue.GEO) geo_pb = field_value_pb.mutable_geo() geo_pb.set_lat(self.value.latitude) geo_pb.set_lng(self.value.longitude)
def _GetFacetValue(value_pb): """Gets the value from the facet value_pb.""" if value_pb.type() == document_pb.FacetValue.ATOM: if value_pb.has_string_value(): return value_pb.string_value() return None if value_pb.type() == document_pb.FieldValue.NUMBER: if value_pb.has_string_value(): return float(value_pb.string_value()) return None raise TypeError('unknown FacetValue type %d' % value_pb.type()) def _GetValue(value_pb): """Gets the value from the value_pb.""" if value_pb.type() in _PROTO_FIELDS_STRING_VALUE: if value_pb.has_string_value(): return value_pb.string_value() return None if value_pb.type() == document_pb.FieldValue.DATE: if value_pb.has_string_value(): return search_util.DeserializeDate(value_pb.string_value()) return None if value_pb.type() == document_pb.FieldValue.NUMBER: if value_pb.has_string_value(): return float(value_pb.string_value()) return None if value_pb.type() == document_pb.FieldValue.GEO: if value_pb.has_geo(): geo_pb = value_pb.geo() return GeoPoint(latitude=geo_pb.lat(), longitude=geo_pb.lng()) return None if value_pb.type() == document_pb.FieldValue.VECTOR: if value_pb.vector_value_size(): return value_pb.vector_value_list() return None raise TypeError('unknown FieldValue type %d' % value_pb.type()) _STRING_TYPES = set([document_pb.FieldValue.TEXT, document_pb.FieldValue.HTML, document_pb.FieldValue.ATOM, document_pb.FieldValue.UNTOKENIZED_PREFIX, document_pb.FieldValue.TOKENIZED_PREFIX]) def _DecodeUTF8(pb_value): """Decodes a UTF-8 encoded string into unicode.""" if pb_value is not None: return pb_value.decode('utf-8') return None def _DecodeValue(pb_value, val_type): """Decodes a possible UTF-8 encoded string value to unicode.""" if val_type in _STRING_TYPES: return _DecodeUTF8(pb_value) return pb_value def _NewFieldFromPb(pb): """Constructs a Field from a document_pb.Field protocol buffer.""" name = _DecodeUTF8(pb.name()) val_type = pb.value().type() value = _DecodeValue(_GetValue(pb.value()), val_type) lang = None if pb.value().has_language(): lang = _DecodeUTF8(pb.value().language()) if val_type == document_pb.FieldValue.TEXT: return TextField(name, value, lang) elif val_type == document_pb.FieldValue.HTML: return HtmlField(name, value, lang) elif val_type == document_pb.FieldValue.ATOM: return AtomField(name, value, lang) elif val_type == document_pb.FieldValue.UNTOKENIZED_PREFIX: return UntokenizedPrefixField(name, value, lang) elif val_type == document_pb.FieldValue.TOKENIZED_PREFIX: return TokenizedPrefixField(name, value, lang) elif val_type == document_pb.FieldValue.DATE: return DateField(name, value) elif val_type == document_pb.FieldValue.NUMBER: return NumberField(name, value) elif val_type == document_pb.FieldValue.GEO: return GeoField(name, value) elif val_type == document_pb.FieldValue.VECTOR: return VectorField(name, value) return InvalidRequest('Unknown field value type %d' % val_type)
[docs]class Document(object): """Represents a user generated document. The following example shows how to create a document consisting of a set of fields, some plain text and some in HTML. Document(doc_id='document_id', fields=[TextField(name='subject', value='going for dinner'), HtmlField(name='body', value='<html>I found a place.</html>'), TextField(name='signature', value='brzydka pogoda', language='pl')], facets=[AtomFacet(name='tag', value='food'), NumberFacet(name='priority', value=5.0)], language='en') """ _FIRST_JAN_2011 = datetime.datetime(2011, 1, 1) def __init__(self, doc_id=None, fields=None, language='en', rank=None, facets=None): """Initializer. Args: doc_id: The visible printable ASCII string identifying the document which does not start with '!'. Whitespace is excluded from ids. If no id is provided, the search service will provide one. fields: An iterable of Field instances representing the content of the document. language: The code of the language used in the field values. rank: The rank of this document used to specify the order in which documents are returned by search. Rank must be a non-negative integer. If not specified, the number of seconds since 1st Jan 2011 is used. Documents are returned in descending order of their rank, in absence of sorting or scoring options. facets: An iterable of Facet instances representing the facets for this document. Raises: TypeError: If any of the parameters have invalid types, or an unknown attribute is passed. ValueError: If any of the parameters have invalid values. """ doc_id = _ConvertToUnicode(doc_id) if doc_id is not None: _CheckDocumentId(doc_id) self._doc_id = doc_id self._fields = _GetList(fields) self._facets = _GetList(facets) self._language = _CheckLanguage(_ConvertToUnicode(language)) self._field_map = None self._facet_map = None if rank is None: rank = self._GetDefaultRank() self._rank_defaulted = True else: self._rank_defaulted = False self._rank = self._CheckRank(rank) _CheckDocument(self) @property def doc_id(self): """Returns the document identifier.""" return self._doc_id @property def fields(self): """Returns a list of fields of the document.""" return self._fields @property def facets(self): """Returns a list of facets of the document.""" return self._facets @property def language(self): """Returns the code of the language the document fields are written in.""" return self._language @property def rank(self): """Returns the rank of this document.""" return self._rank
[docs] def field(self, field_name): """Returns the field with the provided field name. Args: field_name: The name of the field to return. Returns: A field with the given name. Raises: ValueError: There is not exactly one field with the given name. """ fields = self[field_name] if len(fields) == 1: return fields[0] raise ValueError( 'Must have exactly one field with name %s, but found %d.' % (field_name, len(fields)))
[docs] def facet(self, facet_name): """Returns list of facets with the provided name. Args: facet_name: The name of the facet to return. Returns: A list of facets with the given name. """ return self._BuildFacetMap().get(facet_name, [])
def __setstate__(self, state): self.__dict__ = {'_facets': [], '_facet_map': None} self.__dict__.update(state) def __getitem__(self, field_name): """Returns a list of all fields with the provided field name. Args: field_name: The name of the field to return. Returns: All fields with the given name, or an empty list if no field with that name exists. """ return self._BuildFieldMap().get(field_name, []) def __iter__(self): """Documents do not support iteration. This is provided to raise an explicit exception. """ raise TypeError('Documents do not support iteration.') def _BuildFieldMap(self): """Lazily build the field map.""" if self._field_map is None: field_map = {} for field in self._fields: field_map.setdefault(field.name, []).append(field) self._field_map = field_map return self._field_map def _BuildFacetMap(self): """Lazily build the facet map.""" if self._facet_map is None: facet_map = {} for facet in self._facets: facet_map.setdefault(facet.name, []).append(facet) self._facet_map = facet_map return self._facet_map def _CheckRank(self, rank): """Checks if rank is valid, then returns it.""" return _CheckInteger(rank, 'rank', upper_bound=sys.maxint) def _GetDefaultRank(self): """Returns a default rank as total seconds since 1st Jan 2011.""" td = datetime.datetime.now() - Document._FIRST_JAN_2011 return td.seconds + (td.days * 24 * 3600) def __repr__(self): return _Repr( self, [('doc_id', self.doc_id), ('fields', self.fields), ('facets', self.facets), ('language', self.language), ('rank', self.rank)]) def __eq__(self, other): return (isinstance(other, type(self)) and self.doc_id == other.doc_id and self.rank == other.rank and self.language == other.language and self.fields == other.fields and self.facets == other.facets) def __ne__(self, other): return not self == other def __key(self): return self.doc_id def __hash__(self): return hash(self.__key()) def __str__(self): return repr(self)
def _CopyDocumentToProtocolBuffer(document, pb): """Copies Document to a document_pb.Document protocol buffer.""" pb.set_storage(document_pb.Document.DISK) if document.doc_id: pb.set_id(document.doc_id.encode('utf-8')) if document.language: pb.set_language(document.language.encode('utf-8')) for field in document.fields: field_pb = pb.add_field() _CopyFieldToProtocolBuffer(field, field_pb) for facet in document.facets: facet_pb = pb.add_facet() facet._CopyToProtocolBuffer(facet_pb) pb.set_order_id(document.rank) if hasattr(document, '_rank_defaulted'): if document._rank_defaulted: pb.set_order_id_source(document_pb.Document.DEFAULTED) else: pb.set_order_id_source(document_pb.Document.SUPPLIED) return pb def _NewFieldsFromPb(field_list): """Returns a list of Field copied from a document_pb.Document proto buf.""" return [_NewFieldFromPb(f) for f in field_list] def _NewDocumentFromPb(doc_pb): """Constructs a Document from a document_pb.Document protocol buffer.""" lang = None if doc_pb.has_language(): lang = _DecodeUTF8(doc_pb.language()) return Document(doc_id=_DecodeUTF8(doc_pb.id()), fields=_NewFieldsFromPb(doc_pb.field_list()), language=lang, rank=doc_pb.order_id(), facets=_NewFacetsFromPb(doc_pb.facet_list())) def _QuoteString(argument): return '"' + argument.replace('"', '\\\"') + '"'
[docs]class FieldExpression(object): """Represents an expression that will be computed for each result returned. For example, FieldExpression(name='content_snippet', expression='snippet("very important", content)') means a computed field 'content_snippet' will be returned with each search result, which contains HTML snippets of the 'content' field which match the query 'very important'. """ MAXIMUM_EXPRESSION_LENGTH = 1000 MAXIMUM_OPERATOR_LENGTH = 100 def __init__(self, name, expression): """Initializer. Args: name: The name of the computed field for the expression. expression: The expression to evaluate and return in a field with given name in results. See https://developers.google.com/appengine/docs/python/search/overview#Expressions for a list of legal expressions. Raises: TypeError: If any of the parameters has an invalid type, or an unknown attribute is passed. ValueError: If any of the parameters has an invalid value. ExpressionError: If the expression string is not parseable. """ self._name = _CheckFieldName(_ConvertToUnicode(name)) if expression is None: raise ValueError('expression must be a FieldExpression, got None') if not isinstance(expression, basestring): raise TypeError('expression must be a FieldExpression, got %s' % expression.__class__.__name__) self._expression = _CheckExpression(_ConvertToUnicode(expression)) @property def name(self): """Returns name of the expression to return in search results.""" return self._name @property def expression(self): """Returns a string containing an expression returned in search results.""" return self._expression def __repr__(self): return _Repr( self, [('name', self.name), ('expression', self.expression)])
def _CopyFieldExpressionToProtocolBuffer(field_expression, pb): """Copies FieldExpression to a search_service_pb.FieldSpec_Expression.""" pb.set_name(field_expression.name.encode('utf-8')) pb.set_expression(field_expression.expression.encode('utf-8'))
[docs]class SortOptions(object): """Represents a mulit-dimensional sort of Documents. The following code shows how to sort documents based on product rating in descending order and then cheapest product within similarly rated products, sorting at most 1000 documents: SortOptions(expressions=[ SortExpression(expression='rating', direction=SortExpression.DESCENDING, default_value=0), SortExpression(expression='price + tax', direction=SortExpression.ASCENDING, default_value=999999.99)], limit=1000) """ def __init__(self, expressions=None, match_scorer=None, limit=1000): """Initializer. Args: expressions: An iterable of SortExpression representing a multi-dimensional sort of Documents. match_scorer: A match scorer specification which may be used to score documents or in a SortExpression combined with other features. limit: The limit on the number of documents to score or sort. Raises: TypeError: If any of the parameters has an invalid type, or an unknown attribute is passed. ValueError: If any of the parameters has an invalid value. """ self._match_scorer = match_scorer self._expressions = _GetList(expressions) for expression in self._expressions: if not isinstance(expression, SortExpression): raise TypeError('expression must be a SortExpression, got %s' % expression.__class__.__name__) self._limit = _CheckSortLimit(limit) @property def expressions(self): """A list of SortExpression specifying a multi-dimensional sort.""" return self._expressions @property def match_scorer(self): """Returns a match scorer to score documents with.""" return self._match_scorer @property def limit(self): """Returns the limit on the number of documents to score or sort.""" return self._limit def __repr__(self): return _Repr( self, [('match_scorer', self.match_scorer), ('expressions', self.expressions), ('limit', self.limit)])
[docs]class MatchScorer(object): """Assigns a document score based on term frequency. If you add a MatchScorer to a SortOptions as in the following code: sort_opts = search.SortOptions(match_scorer=search.MatchScorer()) then, this will sort the documents in descending score order. The scores will be positive. If you want to sort in ascending order, then use the following code: sort_opts = search.SortOptions(match_scorer=search.MatchScorer(), expressions=[search.SortExpression( expression='_score', direction=search.SortExpression.ASCENDING, default_value=0.0)]) The scores in this case will be negative. """ def __init__(self): """Initializer. Raises: TypeError: If any of the parameters has an invalid type, or an unknown attribute is passed. ValueError: If any of the parameters has an invalid value. """ def __r