diff --git a/.circleci/config.yml b/.circleci/config.yml index fe896a6..18c8708 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,8 @@ orbs: linter: thekevjames/linter@0.1 jobs: - test: + # TODO: Remove old_nox when everything is split into modules + old_nox: docker: - image: thekevjames/nox:2019.5.30 environment: @@ -14,6 +15,19 @@ jobs: - checkout - run: nox + nox: + docker: + - image: thekevjames/nox:2019.5.30 + environment: + GOOGLE_APPLICATION_CREDENTIALS: /key.json + parameters: + folder: + type: string + steps: + - run: echo ${GOOGLE_SERVICE_PUBLIC} | base64 -d > ${GOOGLE_APPLICATION_CREDENTIALS} + - checkout + - run: nox -f <>/noxfile.py + pypi: docker: - image: python:3.7.3-slim @@ -23,6 +37,13 @@ jobs: - deploy: name: upload to pypi command: | + # TODO: Simplify when everything is split into modules + # The check for the dash in the tag name won't be necessary + if [[ $CIRCLE_TAG =~ "-" ]] + then + export PROJECT=$(echo "${CIRCLE_TAG}" | sed 's/-.*//') + cd "${PROJECT}" + fi python setup.py sdist bdist_wheel twine upload dist/* @@ -41,13 +62,27 @@ jobs: - deploy: name: create GitHub release command: | - export PREV_RELEASE=$(git tag --sort=version:refname | tail -n2 | head -n1) - [ "${PREV_RELEASE}" = "${CIRCLE_TAG}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD) - [ -z "${PREV_RELEASE}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD) + # TODO: Simplify when everything is split into modules + # The check for the dash in the tag name won't be necessary + if [[ $CIRCLE_TAG =~ "-" ]] + then + export PROJECT=$(echo "${CIRCLE_TAG}" | sed 's/-.*//') + export PREV_RELEASE=$(git tag --sort=version:refname | grep ${PROJECT} | tail -n2 | head -n1) + [ "${PREV_RELEASE}" = "${CIRCLE_TAG}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD) + [ -z "${PREV_RELEASE}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD) + + git log ${PREV_RELEASE}..${CIRCLE_TAG} --pretty=format:'- %s' > release-description.md + ./bin/linux/amd64/github-release release -t "${CIRCLE_TAG}" + cat release-description.md | grep ${PROJECT} | ./bin/linux/amd64/github-release edit -t ${CIRCLE_TAG} -d - + else + export PREV_RELEASE=$(git tag --sort=version:refname | tail -n2 | head -n1) + [ "${PREV_RELEASE}" = "${CIRCLE_TAG}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD) + [ -z "${PREV_RELEASE}" ] && export PREV_RELEASE=$(git rev-list --max-parents=0 HEAD) - git log ${PREV_RELEASE}..${CIRCLE_TAG} --pretty=format:'- %s' > release-description.md - ./bin/linux/amd64/github-release release -t "${CIRCLE_TAG}" - cat release-description.md | ./bin/linux/amd64/github-release edit -t ${CIRCLE_TAG} -d - + git log ${PREV_RELEASE}..${CIRCLE_TAG} --pretty=format:'- %s' > release-description.md + ./bin/linux/amd64/github-release release -t "${CIRCLE_TAG}" + cat release-description.md | ./bin/linux/amd64/github-release edit -t ${CIRCLE_TAG} -d - + fi workflows: run-jobs: @@ -78,7 +113,14 @@ workflows: tags: only: /.*/ - - test: + - old_nox: + filters: + tags: + only: /.*/ + + - nox: + name: test-datastore + folder: datastore filters: tags: only: /.*/ @@ -89,23 +131,27 @@ workflows: branches: ignore: /.*/ tags: - only: /[0-9]+\.[0-9]+\.[0-9]+/ + # TODO make project name mandatory when split into modules + only: /([a-z]+-)?[0-9]+\.[0-9]+\.[0-9]+/ requires: - lint-py27 - lint-py35 - lint-py36 - lint-py37 - - test + - old_nox + - test-datastore - github: context: org-global filters: branches: ignore: /.*/ tags: - only: /[0-9]+\.[0-9]+\.[0-9]+/ + # TODO make project name mandatory when split into modules + only: /([a-z]+-)?[0-9]+\.[0-9]+\.[0-9]+/ requires: - lint-py27 - lint-py35 - lint-py36 - lint-py37 - - test + - old_nox + - test-datastore diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index cc6dd88..a984c53 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -5,14 +5,29 @@ Thanks for contributing to ``gcloud-rest``! We appreciate contributions of any size and hope to make it easy for you to dive in. Here's the thousand-foot overview of how we've set this project up. +Project Setup +------------- + +Our vision is that each module (``auth``, ``core``, ``kms``, etc.) should be +released in its own PyPI package. + +Some modules are still bundled together in the ``gcloud-rest`` package. If you +are making changes to one of those modules, please consider creating another +PR to create a separate package for it. You can get inspiration from the +`datastore module`_. + Testing ------- -Tests are run with `nox`_. See ``noxfile.py`` for the scaffolding and the -``tests/unit`` and ``tests/integration`` folders for the actual test code. +Tests are run with `nox`_. See each project's ``noxfile.py`` for the scaffolding +and the ``tests/unit`` and ``tests/integration`` folders for the actual test +code. + +You can get nox with ``pip install nox`` and run a specific project's tests with +``nox -f /noxfile.py``. -You can get nox with ``pip install nox`` and run the project's tests with -``nox``. +Modules in the ``gcloud-rest`` package are tested together, using the root +``noxfile.py``. You can run their tests by running ``nox``. Local Development ~~~~~~~~~~~~~~~~~ @@ -72,6 +87,31 @@ in CI against all changesets. You can also run ``pre-commit`` in an ad-hoc fashion by calling ``pre-commit run --all-files``. +CircleCI will also make sure that the code is correct in Python 2.7. +To check that locally, run ``pre-commit run -c .pre-commit-config.py27.yaml --all-files`` +in a Python 2.7 environment. + +It may be useful to have two Python virtual environments and run pre-commit in both. + +Python 2.7: + +.. code-block:: console + + $ virtualenv venv-27 + $ source venv-27/bin/activate + $ pip install pre-commit + $ pre-commit run -c .pre-commit-config.py27.yaml --all-files + +Python 3.7: + +.. code-block:: console + + $ python3 -m venv venv-37 + $ source venv-37/bin/activate + $ pip install pre-commit + $ pre-commit run --all-files + + Other than the above enforced standards, we like code that is easy-to-read for any new or returning contributors with relevant comments where appropriate. @@ -82,6 +122,7 @@ If you are a maintainer looking to release a new version, see our `Release documentation`_. .. _conventional changelog: https://github.com/conventional-changelog/conventional-changelog +.. _datastore module: https://github.com/talkiq/gcloud-rest/blob/master/datastore .. _nox: https://nox.readthedocs.io/en/latest/ .. _pre-commit: http://pre-commit.com/ .. _Pull Request: https://github.com/talkiq/gcloud-rest/pull/new/master diff --git a/.gitignore b/.gitignore index 7a59d18..8318d5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ -**/*.pyc +**/*.egg-info/* +**/.idea/* +**/.nox/* +**/.pytest_cache/* **/__pycache__/* -*.egg-info/* -.cache/* -.coverage* -.idea/* -.nox/* -build/* -dist/* -venv/* +**/dist/* +**/venv*/* +*.pyc +*/.coverage* +*/build/* diff --git a/.pre-commit-config.py27.yaml b/.pre-commit-config.py27.yaml index 0149127..264fc79 100644 --- a/.pre-commit-config.py27.yaml +++ b/.pre-commit-config.py27.yaml @@ -39,7 +39,7 @@ repos: - -d locally-disabled - -d missing-docstring - -d too-few-public-methods - - -d too-many-arguments + - -d too-many-arguments # TODO stop ignoring this and fix the code - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.6 hooks: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75486f7..587e0d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,6 +41,7 @@ repos: - -d too-few-public-methods - -d too-many-arguments - -d useless-object-inheritance # necessary for Python 2 compatibility + - -d too-many-arguments # TODO stop ignoring this and fix the code - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.6 hooks: diff --git a/README.rst b/README.rst index 2e337df..9a3516d 100644 --- a/README.rst +++ b/README.rst @@ -11,6 +11,14 @@ support for ``asyncio``. |pypi| |circleci| |coverage| |pythons| +Some of the client libraries live in separate packages: + +- |pypids| `Google Cloud Datastore`_ (`Datastore README`_) + +Note that all the modules will eventually be released separately, and the +``gcloud-rest`` package will be deprecated. + + Installation ------------ @@ -21,8 +29,13 @@ Installation Usage ----- -This project currently exposes interfaces to ``CloudTasks``, ``KMS``, and -``Storage``. +This project currently exposes interfaces to ``CloudTasks``, ``Datastore``, +``KMS``, and ``Storage``. + +For the libraries that are packaged separately, you can find usage examples +in the corresponding README file. + +For the others, keep reading. Storage (see `bucket.py`_): @@ -117,6 +130,9 @@ TaskManager (for ``CloudTasks``, see `manager.py`_): tm = TaskManager('my-project', 'taskqueue-name', worker_method) tm.find_tasks_forever() +.. _Google Cloud Datastore: https://pypi.org/project/gcloud-rest-datastore/ +.. _Datastore README: https://github.com/talkiq/gcloud-rest/blob/master/datastore/README.rst + .. _bucket.py: https://github.com/talkiq/gcloud-rest/blob/master/gcloud/rest/storage/bucket.py .. _client.py: https://github.com/talkiq/gcloud-rest/blob/master/gcloud/rest/kms/client.py .. _manager.py: https://github.com/talkiq/gcloud-rest/blob/master/gcloud/rest/taskqueue/manager.py @@ -139,3 +155,7 @@ TaskManager (for ``CloudTasks``, see `manager.py`_): .. |pythons| image:: https://img.shields.io/pypi/pyversions/gcloud-rest.svg?style=flat-square :alt: Python Version Support :target: https://pypi.org/project/gcloud-rest/ + +.. |pypids| image:: https://img.shields.io/pypi/v/gcloud-rest-datastore.svg?style=flat-square + :alt: Latest PyPI Version + :target: https://pypi.org/project/gcloud-rest-datastore/ diff --git a/datastore/LICENSE b/datastore/LICENSE new file mode 100644 index 0000000..d52ce2f --- /dev/null +++ b/datastore/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 TalkIQ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/datastore/MANIFEST.in b/datastore/MANIFEST.in new file mode 100644 index 0000000..791d443 --- /dev/null +++ b/datastore/MANIFEST.in @@ -0,0 +1,4 @@ +include README.rst LICENSE requirements.txt +recursive-include gcloud *.py +recursive-include tests *.py +global-exclude *.pyc __pycache__ diff --git a/datastore/README.rst b/datastore/README.rst new file mode 100644 index 0000000..369f275 --- /dev/null +++ b/datastore/README.rst @@ -0,0 +1,184 @@ +RESTful Python Client for Google Cloud Datastore +================================================ + +|pypi| |pythons| + +Installation +------------ + +.. code-block:: console + + $ pip install --upgrade gcloud-rest-datastore + +Usage +----- + +We're still working on documentation; for now, this should help get you +started: + +.. code-block:: python + + from gcloud.rest.datastore import Datastore + from gcloud.rest.datastore import Direction + from gcloud.rest.datastore import Filter + from gcloud.rest.datastore import GQLQuery + from gcloud.rest.datastore import Key + from gcloud.rest.datastore import PathElement + from gcloud.rest.datastore import PropertyFilter + from gcloud.rest.datastore import PropertyFilterOperator + from gcloud.rest.datastore import PropertyOrder + from gcloud.rest.datastore import Query + from gcloud.rest.datastore import Value + + ds = Datastore('my-gcloud-project', '/path/to/creds.json') + key1 = Key('my-gcloud-project', [PathElement('Kind', 'entityname')]) + key2 = Key('my-gcloud-project', [PathElement('Kind', 'entityname2')]) + + # batched lookups + entities = ds.lookup([key1, key2]) + + # convenience functions for any datastore mutations + ds.insert(key1, {'a_boolean': True, 'meaning_of_life': 41}) + ds.update(key1, {'a_boolean': True, 'meaning_of_life': 42}) + ds.upsert(key1, {'animal': 'aardvark'}) + ds.delete(key1) + + # or build your own mutation sequences with full transaction support + transaction = ds.beginTransaction() + try: + mutations = [ + ds.make_mutation(Operation.INSERT, key1, properties={'animal': 'sloth'}), + ds.make_mutation(Operation.UPSERT, key1, properties={'animal': 'aardvark'}), + ds.make_mutation(Operation.INSERT, key2, properties={'animal': 'aardvark'}), + ] + ds.commit(transaction, mutations=[mutation]) + except Exception: + ds.rollback(transaction) + + # support for partial keys + partial_key = Key('my-gcloud-project', [PathElement('Kind')]) + # and ID allocation or reservation + allocated_keys = ds.allocateIds([partial_key]) + ds.reserveIds(allocated_keys) + + # query support + property_filter = PropertyFilter(prop='answer', + operator=PropertyFilterOperator.EQUAL, + value=Value(42)) + property_order = PropertyOrder(prop='length', + direction=Direction.DESCENDING) + query = Query(kind='the_meaning_of_life', + query_filter=Filter(property_filter), + order=property_order) + results = ds.runQuery(query, session=s) + + # alternatively, query support using GQL + gql_query = GQLQuery('SELECT * FROM the_meaning_of_life WHERE answer = @answer', + named_bindings={'answer': 42}) + results = ds.runQuery(gql_query, session=s) + +Custom Subclasses +~~~~~~~~~~~~~~~~~ + +``gcloud-rest-datastore`` provides class interfaces mirroring all official +Google API types, ie. ``Key`` and ``PathElement``, ``Entity`` and +``EntityResult``, ``QueryResultBatch``, and ``Value``. These types will be +returned from arbitrary Datastore operations, for example +``Datastore.allocateIds(...)`` will return a list of ``Key`` entities. + +For advanced usage, all of these datatypes may be overloaded. A common use-case +may be to deserialize entities into more specific classes. For example, given a +custom entity class such as: + +.. code-block:: python + + class MyEntityKind(gcloud.rest.datastore.Entity): + def __init__(self, key, properties = None) -> None: + self.key = key + self.is_an_aardvark = (properties or {}).get('aardvark', False) + + def __repr__(self): + return "I'm an aardvark!" if self.is_an_aardvark else "Sorry, nope" + +We can then configure ``gcloud-rest-datastore`` to serialize/deserialize from +this custom entity class with: + +.. code-block:: python + + class MyCustomDatastore(gcloud.rest.datastore.Datastore): + entity_result_kind.entity_kind = MyEntityKind + +The full list of classes which may be overridden in this way is: + +.. code-block:: python + + class MyVeryCustomDatastore(gcloud.rest.datastore.Datastore): + datastore_operation_kind = DatastoreOperation + entity_result_kind = EntityResult + entity_result_kind.entity_kind = Entity + entity_result_kind.entity_kind.key_kind = Key + key_kind = Key + key_kind.path_element_kind = PathElement + query_result_batch_kind = QueryResultBatch + query_result_batch_kind.entity_result_kind = EntityResult + value_kind = Value + value_kind.key_kind = Key + + class MyVeryCustomQuery(gcloud.rest.datastore.Query): + value_kind = Value + + class MyVeryCustomGQLQuery(gcloud.rest.datastore.GQLQuery): + value_kind = Value + +You can then drop-in the ``MyVeryCustomDatastore`` class anywhere where you +previously used ``Datastore`` and do the same for ``Query`` and ``GQLQuery``. + +To override any sub-key, you'll need to override any parents which use it. For +example, if you want to use a custom Key kind and be able to use queries with +it, you will need to implement your own ``Value``, ``Query``, and ``GQLQuery`` +classes and wire them up to the rest of the custom classes: + +.. code-block:: python + + class MyKey(gcloud.rest.datastore.Key): + pass + + class MyValue(gcloud.rest.datastore.Value): + key_kind = MyKey + + class MyEntity(gcloud.rest.datastore.Entity): + key_kind = MyKey + value_kind = MyValue + + class MyEntityResult(gcloud.rest.datastore.EntityResult): + entity_kind = MyEntity + + class MyQueryResultBatch(gcloud.rest.datastore.QueryResultBatch): + entity_result_kind = MyEntityResult + + class MyDatastore(gcloud.rest.datastore.Datastore): + key_kind = MyKey + entity_result_kind = MyEntityResult + query_result_batch = MyQueryResultBatch + value_kind = MyValue + + class MyQuery(gcloud.rest.datastore.Query): + value_kind = MyValue + + class MyGQLQuery(gcloud.rest.datastore.GQLQuery): + value_kind = MyValue + +Contributing +------------ + +Please see our `contributing guide`_. + +.. _contributing guide: https://github.com/talkiq/gcloud-rest/blob/master/.github/CONTRIBUTING.rst + +.. |pypi| image:: https://img.shields.io/pypi/v/gcloud-rest-datastore.svg?style=flat-square + :alt: Latest PyPI Version + :target: https://pypi.org/project/gcloud-rest-datastore/ + +.. |pythons| image:: https://img.shields.io/pypi/pyversions/gcloud-rest-datastore.svg?style=flat-square + :alt: Python Version Support + :target: https://pypi.org/project/gcloud-rest-datastore/ diff --git a/datastore/gcloud/__init__.py b/datastore/gcloud/__init__.py new file mode 100644 index 0000000..267f710 --- /dev/null +++ b/datastore/gcloud/__init__.py @@ -0,0 +1,6 @@ +try: + import pkg_resources + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/datastore/gcloud/rest/__init__.py b/datastore/gcloud/rest/__init__.py new file mode 100644 index 0000000..267f710 --- /dev/null +++ b/datastore/gcloud/rest/__init__.py @@ -0,0 +1,6 @@ +try: + import pkg_resources + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/datastore/gcloud/rest/datastore/__init__.py b/datastore/gcloud/rest/datastore/__init__.py new file mode 100644 index 0000000..e11280d --- /dev/null +++ b/datastore/gcloud/rest/datastore/__init__.py @@ -0,0 +1,34 @@ +from pkg_resources import get_distribution +__version__ = get_distribution('gcloud-rest').version + +from gcloud.rest.datastore.constants import CompositeFilterOperator +from gcloud.rest.datastore.constants import Consistency +from gcloud.rest.datastore.constants import Direction +from gcloud.rest.datastore.constants import Mode +from gcloud.rest.datastore.constants import MoreResultsType +from gcloud.rest.datastore.constants import Operation +from gcloud.rest.datastore.constants import PropertyFilterOperator +from gcloud.rest.datastore.constants import ResultType +from gcloud.rest.datastore.datastore import Datastore +from gcloud.rest.datastore.datastore import SCOPES +from gcloud.rest.datastore.datastore_operation import DatastoreOperation +from gcloud.rest.datastore.entity import Entity +from gcloud.rest.datastore.entity import EntityResult +from gcloud.rest.datastore.filter import CompositeFilter +from gcloud.rest.datastore.filter import Filter +from gcloud.rest.datastore.filter import PropertyFilter +from gcloud.rest.datastore.key import Key +from gcloud.rest.datastore.key import PathElement +from gcloud.rest.datastore.property_order import PropertyOrder +from gcloud.rest.datastore.query import GQLQuery +from gcloud.rest.datastore.query import Query +from gcloud.rest.datastore.query import QueryResultBatch +from gcloud.rest.datastore.value import Value + + +__all__ = ['__version__', 'CompositeFilter', 'CompositeFilterOperator', + 'Consistency', 'Datastore', 'DatastoreOperation', 'Direction', + 'Entity', 'EntityResult', 'Filter', 'GQLQuery', 'Key', 'Mode', + 'MoreResultsType', 'Operation', 'PathElement', 'PropertyFilter', + 'PropertyFilterOperator', 'PropertyOrder', 'Query', + 'QueryResultBatch', 'ResultType', 'SCOPES', 'Value'] diff --git a/datastore/gcloud/rest/datastore/constants.py b/datastore/gcloud/rest/datastore/constants.py new file mode 100644 index 0000000..4bfbfca --- /dev/null +++ b/datastore/gcloud/rest/datastore/constants.py @@ -0,0 +1,80 @@ +import enum +from datetime import datetime as dt + + +class CompositeFilterOperator(enum.Enum): + AND = 'AND' + UNSPECIFIED = 'OPERATOR_UNSPECIFIED' + + +class Consistency(enum.Enum): + EVENTUAL = 'EVENTUAL' + STRONG = 'STRONG' + UNSPECIFIED = 'READ_CONSISTENCY_UNSPECIFIED' + + +class Direction(enum.Enum): + ASCENDING = 'ASCENDING' + DESCENDING = 'DESCENDING' + UNSPECIFIED = 'DIRECTION_UNSPECIFIED' + + +class Mode(enum.Enum): + NON_TRANSACTIONAL = 'NON_TRANSACTIONAL' + TRANSACTIONAL = 'TRANSACTIONAL' + UNSPECIFIED = 'MODE_UNSPECIFIED' + + +class MoreResultsType(enum.Enum): + MORE_RESULTS_AFTER_CURSOR = 'MORE_RESULTS_AFTER_CURSOR' + MORE_RESULTS_AFTER_LIMIT = 'MORE_RESULTS_AFTER_LIMIT' + NO_MORE_RESULTS = 'NO_MORE_RESULTS' + NOT_FINISHED = 'NOT_FINISHED' + UNSPECIFIED = 'MORE_RESULTS_TYPE_UNSPECIFIED' + + +class Operation(enum.Enum): + DELETE = 'delete' + INSERT = 'insert' + UPDATE = 'update' + UPSERT = 'upsert' + + +class PropertyFilterOperator(enum.Enum): + EQUAL = 'EQUAL' + GREATER_THAN = 'GREATER_THAN' + GREATER_THAN_OR_EQUAL = 'GREATER_THAN_OR_EQUAL' + HAS_ANCESTOR = 'HAS_ANCESTOR' + LESS_THAN = 'LESS_THAN' + LESS_THAN_OR_EQUAL = 'LESS_THAN_OR_EQUAL' + UNSPECIFIED = 'OPERATOR_UNSPECIFIED' + + +class ResultType(enum.Enum): + FULL = 'FULL' + KEY_ONLY = 'KEY_ONLY' + PROJECTION = 'PROJECTION' + UNSPECIFIED = 'RESULT_TYPE_UNSPECIFIED' + + +class TypeName(enum.Enum): + BLOB = 'blobValue' + BOOLEAN = 'booleanValue' + DOUBLE = 'doubleValue' + INTEGER = 'integerValue' + KEY = 'keyValue' + NULL = 'nullValue' + STRING = 'stringValue' + TIMESTAMP = 'timestampValue' + + +# TODO: add types for geoPointValue and arrayValue +TYPES = { + bool: TypeName.BOOLEAN, + bytes: TypeName.BLOB, + dt: TypeName.TIMESTAMP, + float: TypeName.DOUBLE, + int: TypeName.INTEGER, + str: TypeName.STRING, + type(None): TypeName.NULL, +} diff --git a/datastore/gcloud/rest/datastore/datastore.py b/datastore/gcloud/rest/datastore/datastore.py new file mode 100644 index 0000000..ba14282 --- /dev/null +++ b/datastore/gcloud/rest/datastore/datastore.py @@ -0,0 +1,430 @@ +import logging +import os +import threading +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import +from typing import Union # pylint: disable=unused-import + +import requests + +from gcloud.rest.auth import Token # pylint: disable=no-name-in-module +from gcloud.rest.datastore.constants import Consistency +from gcloud.rest.datastore.constants import Mode +from gcloud.rest.datastore.constants import Operation +from gcloud.rest.datastore.datastore_operation import DatastoreOperation +from gcloud.rest.datastore.entity import EntityResult +from gcloud.rest.datastore.key import Key +from gcloud.rest.datastore.query import BaseQuery # pylint: disable=unused-import +from gcloud.rest.datastore.query import QueryResultBatch +from gcloud.rest.datastore.value import Value +try: + import ujson as json +except ImportError: + import json # type: ignore + + +try: + API_ROOT = 'http://%s/v1' % os.environ['DATASTORE_EMULATOR_HOST'] + IS_DEV = True +except KeyError: + API_ROOT = 'https://datastore.googleapis.com/v1' + IS_DEV = False + +SCOPES = [ + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/datastore', +] + +log = logging.getLogger(__name__) + + +class Datastore(object): + datastore_operation_kind = DatastoreOperation + entity_result_kind = EntityResult + key_kind = Key + query_result_batch_kind = QueryResultBatch + value_kind = Value + + def __init__(self, + project=None, # type: Optional[str] + service_file=None, # type: Optional[str] + namespace='', # type: str + session=None, # type: Optional[requests.Session] + token=None, # type: Optional[Token] + google_api_lock=None, # type: Optional[threading.RLock] + ): + # type: (...) -> None + self.namespace = namespace + self.session = session + self.google_api_lock = google_api_lock or threading.RLock() + + if IS_DEV: + self._project = os.environ.get('DATASTORE_PROJECT_ID', 'dev') + # Tokens are not needed when using dev emulator + self.token = None + else: + self._project = project + self.token = token or Token(service_file=service_file, + session=session, scopes=SCOPES) + + def project(self): + # type: () -> str + if self._project: + return self._project + + self._project = self.token.get_project() + if self._project: + return self._project + + raise Exception('could not determine project, please set it manually') + + @staticmethod + def _make_commit_body(mutations, transaction=None, + mode=Mode.TRANSACTIONAL): + # type: (List[Dict[str, Any]], Optional[str], Mode) -> Dict[str, Any] + if not mutations: + raise Exception('at least one mutation record is required') + + if transaction is None and mode != Mode.NON_TRANSACTIONAL: + raise Exception('a transaction ID must be provided when mode is ' + 'transactional') + + data = { + 'mode': mode.value, + 'mutations': mutations, + } + if transaction is not None: + data['transaction'] = transaction + return data + + def headers(self): + # type: () -> Dict[str, str] + if IS_DEV: + return {} + + token = self.token.get() + return { + 'Authorization': 'Bearer %s' % token, + } + + # TODO: support mutations w version specifiers, return new version (commit) + @classmethod + def make_mutation(cls, operation, key, properties=None): + # type: (Operation, Key, Optional[Dict[str, Any]]) -> Dict[str, Any] + if operation == Operation.DELETE: + return {operation.value: key.to_repr()} + + return { + operation.value: { + 'key': key.to_repr(), + 'properties': {k: cls.value_kind(v).to_repr() + for k, v in properties.items()}, + } + } + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/allocateIds + def allocateIds(self, keys, session=None, timeout=10): + # type: (List[Key], Optional[requests.Session], int) -> List[Key] + + project = self.project() + url = '%s/projects/%s:allocateIds' % (API_ROOT, project) + + payload = json.dumps({ + 'keys': [k.to_repr() for k in keys], + }).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + data = resp.json() # type: dict + + return [self.key_kind.from_repr(k) for k in data['keys']] + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/beginTransaction + # TODO: support readwrite vs readonly transaction types + def beginTransaction(self, session=None, timeout=10): + # type: (requests.Session, int) -> str + project = self.project() + url = '%s/projects/%s:beginTransaction' % (API_ROOT, project) + headers = self.headers() + headers.update({ + 'Content-Length': '0', + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, headers=headers, timeout=timeout) + resp.raise_for_status() + data = resp.json() + + transaction = data['transaction'] # type: str + return transaction + + # TODO: return mutation results + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/commit + def commit(self, + mutations, # type: List[Dict[str, Any]] + transaction=None, # type: Optional[str] + mode=Mode.TRANSACTIONAL, # type: Mode + session=None, # type: Optional[requests.Session] + timeout=1 # type: int + ): + # type: (...) -> None + project = self.project() + url = '%s/projects/%s:commit' % (API_ROOT, project) + + body = self._make_commit_body(mutations, transaction=transaction, + mode=mode) + payload = json.dumps(body).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + + # https://cloud.google.com/datastore/docs/reference/admin/rest/v1/projects/export + def export(self, + output_bucket_prefix, # type: str + kinds=None, # type: Optional[List[str]] + namespaces=None, # type: Optional[List[str]] + labels=None, # type: Optional[Dict[str, str]] + session=None, # type: Optional[requests.Session] + timeout=10 # type: int + ): + # type: (...) -> DatastoreOperation + project = self.project() + url = '%s/projects/%s:export' % (API_ROOT, project) + + payload = json.dumps({ + 'entityFilter': { + 'kinds': kinds or [], + 'namespaceIds': namespaces or [], + }, + 'labels': labels or {}, + 'outputUrlPrefix': 'gs://' + output_bucket_prefix, + }).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + data = resp.json() # type: dict + + return self.datastore_operation_kind.from_repr(data) + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects.operations/get + def get_datastore_operation(self, name, session, timeout): + # type: (str, requests.Session, int) -> DatastoreOperation + url = '%s/%s' % (API_ROOT, name) + + headers = self.headers() + headers.update({ + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.get(url, headers=headers, timeout=timeout) + resp.raise_for_status() + data = resp.json() # type: dict + + return self.datastore_operation_kind.from_repr(data) + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/lookup + def lookup(self, + keys, # type: List[Key] + transaction=None, # type: Optional[str] + consistency=Consistency.STRONG, # type: Consistency + session=None, # type: Optional[requests.Session] + timeout=10 # type: int + ): + # type: (...) -> Dict[str, Union[EntityResult, Key]] + project = self.project() + url = '%s/projects/%s:lookup' % (API_ROOT, project) + + if transaction: + options = {'transaction': transaction} + else: + options = {'readConsistency': consistency.value} + payload = json.dumps({ + 'keys': [k.to_repr() for k in keys], + 'readOptions': options, + }).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + data = resp.json() # type: dict + + return { + 'found': [self.entity_result_kind.from_repr(e) + for e in data.get('found', [])], + 'missing': [self.entity_result_kind.from_repr(e) + for e in data.get('missing', [])], + 'deferred': [self.key_kind.from_repr(k) + for k in data.get('deferred', [])], + } + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/reserveIds + def reserveIds(self, keys, database_id='', session=None, timeout=10): + # type (List[Key], str, Optional[requests.Session], int) -> None + project = self.project() + url = '%s/projects/%s:reserveIds' % (API_ROOT, project) + + payload = json.dumps({ + 'databaseId': database_id, + 'keys': [k.to_repr() for k in keys], + }).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/rollback + def rollback(self, transaction, session=None, timeout=10): + # type: (str, requests.Session, int) -> None + project = self.project() + url = '%s/projects/%s:rollback' % (API_ROOT, project) + + payload = json.dumps({ + 'transaction': transaction, + }).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + + # https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery + def runQuery(self, + query, # type: BaseQuery + transaction=None, # type: Optional[str] + consistency=Consistency.EVENTUAL, # type: Consistency + session=None, # type: Optional[requests.Session] + timeout=10 # type: int + ): + # type (...) -> QueryResultBatch + project = self.project() + url = '%s/projects/%s:runQuery' % (API_ROOT, project) + + if transaction: + options = {'transaction': transaction} + else: + options = {'readConsistency': consistency.value} + payload = json.dumps({ + 'partitionId': { + 'projectId': project, + 'namespaceId': self.namespace, + }, + query.json_key: query.to_repr(), + 'readOptions': options, + }).encode('utf-8') + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(payload)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=payload, + headers=headers, timeout=timeout) + resp.raise_for_status() + + data = resp.json() # type: dict + return self.query_result_batch_kind.from_repr(data['batch']) + + def delete(self, key, session=None): + # type: (Key, Optional[requests.Session]) -> None + return self.operate(Operation.DELETE, key, session=session) + + def insert(self, key, properties, session=None): + # type: (Key, Dict[str, Any], Optional[requests.Session]) -> None + return self.operate(Operation.INSERT, key, properties, session=session) + + def update(self, key, properties, session=None): + # type: (Key, Dict[str, Any], Optional[requests.Session]) -> None + return self.operate(Operation.UPDATE, key, properties, session=session) + + def upsert(self, key, properties, session=None): + # type: (Key, Dict[str, Any], Optional[requests.Session]) -> None + return self.operate(Operation.UPSERT, key, properties, session=session) + + # TODO: accept Entity rather than key/properties? + def operate(self, + operation, # type: Operation + key, # type: Key + properties=None, # type: Optional[Dict[str, Any]] + session=None # type: Optional[requests.Session] + ): + # type (...) -> None + transaction = self.beginTransaction(session=session) + mutation = self.make_mutation(operation, key, properties=properties) + self.commit([mutation], transaction=transaction, session=session) diff --git a/datastore/gcloud/rest/datastore/datastore_operation.py b/datastore/gcloud/rest/datastore/datastore_operation.py new file mode 100644 index 0000000..ddbfdc3 --- /dev/null +++ b/datastore/gcloud/rest/datastore/datastore_operation.py @@ -0,0 +1,36 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import + + +class DatastoreOperation(object): + def __init__(self, + name, # type: str + done, # type: bool + metadata=None, # type: Optional[Dict[str, Any]] + error=None, # type: Optional[Dict[str, str]] + response=None # type: Optional[Dict[str, Any]] + ): + # type: (...) -> None + self.name = name + self.done = done + + self.metadata = metadata + self.error = error + self.response = response + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> DatastoreOperation + return cls(data['name'], data['done'], data.get('metadata'), + data.get('error'), data.get('response')) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'done': self.done, + 'error': self.error, + 'metadata': self.metadata, + 'name': self.name, + 'response': self.response, + } diff --git a/datastore/gcloud/rest/datastore/entity.py b/datastore/gcloud/rest/datastore/entity.py new file mode 100644 index 0000000..2060161 --- /dev/null +++ b/datastore/gcloud/rest/datastore/entity.py @@ -0,0 +1,81 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import + +from gcloud.rest.datastore.key import Key +from gcloud.rest.datastore.value import Value + + +class Entity(object): + key_kind = Key + value_kind = Value + + def __init__(self, key, properties=None): + # type: (Key, Optional[Dict[str, dict]]) -> None + self.key = key + self.properties = {k: self.value_kind.from_repr(v).value + for k, v in (properties or {}).items()} + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, Entity): + return False + + return bool(self.key == other.key + and self.properties == other.properties) + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> Entity + return cls(cls.key_kind.from_repr(data['key']), data.get('properties')) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'key': self.key.to_repr(), + 'properties': self.properties, + } + + +class EntityResult(object): + entity_kind = Entity + + def __init__(self, entity, version, cursor=''): + # type: (Entity, str, str) -> None + self.entity = entity + self.version = version + self.cursor = cursor + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, EntityResult): + return False + + return bool(self.entity == other.entity + and self.version == other.version + and self.cursor == self.cursor) + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> EntityResult + return cls(cls.entity_kind.from_repr(data['entity']), data['version'], + data.get('cursor', '')) + + def to_repr(self): + # type: () -> Dict[str, Any] + data = { + 'entity': self.entity.to_repr(), + 'version': self.version, + } + if self.cursor: + data['cursor'] = self.cursor + + return data diff --git a/datastore/gcloud/rest/datastore/filter.py b/datastore/gcloud/rest/datastore/filter.py new file mode 100644 index 0000000..1ed5b5e --- /dev/null +++ b/datastore/gcloud/rest/datastore/filter.py @@ -0,0 +1,128 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import + +from gcloud.rest.datastore.constants import CompositeFilterOperator +from gcloud.rest.datastore.constants import PropertyFilterOperator +from gcloud.rest.datastore.value import Value + + +class BaseFilter(object): + json_key = '' + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> BaseFilter + raise NotImplementedError + + def to_repr(self): + # type: () -> Dict[str, Any] + raise NotImplementedError + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#Filter +class Filter(object): + def __init__(self, inner_filter): + # type: (BaseFilter) -> None + self.inner_filter = inner_filter + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, Filter): + return False + + return self.inner_filter == other.inner_filter + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> Filter + if 'compositeFilter' in data: + return cls(CompositeFilter.from_repr(data['compositeFilter'])) + if 'propertyFilter' in data: + return cls(PropertyFilter.from_repr(data['propertyFilter'])) + + raise ValueError('invalid filter name: %s' % data.keys()) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + self.inner_filter.json_key: self.inner_filter.to_repr(), + } + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#CompositeFilter +class CompositeFilter(BaseFilter): + json_key = 'compositeFilter' + + def __init__(self, operator, filters): + # type: (CompositeFilterOperator, List[Filter]) -> None + self.operator = operator + self.filters = filters + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, CompositeFilter): + return False + + return bool( + self.operator == other.operator + and self.filters == other.filters) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> CompositeFilter + operator = CompositeFilterOperator(data['op']) + filters = [Filter.from_repr(f) for f in data['filters']] + return cls(operator=operator, filters=filters) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'filters': [f.to_repr() for f in self.filters], + 'op': self.operator.value, + } + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#PropertyFilter +class PropertyFilter(BaseFilter): + json_key = 'propertyFilter' + + def __init__(self, prop, operator, value): + # type: (str, PropertyFilterOperator, Value) -> None + self.prop = prop + self.operator = operator + self.value = value + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, PropertyFilter): + return False + + return bool( + self.prop == other.prop + and self.operator == other.operator + and self.value == other.value) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> PropertyFilter + prop = data['property']['name'] + operator = PropertyFilterOperator(data['op']) + value = Value.from_repr(data['value']) + return cls(prop=prop, operator=operator, value=value) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'op': self.operator.value, + 'property': {'name': self.prop}, + 'value': self.value.to_repr(), + } diff --git a/datastore/gcloud/rest/datastore/key.py b/datastore/gcloud/rest/datastore/key.py new file mode 100644 index 0000000..533aead --- /dev/null +++ b/datastore/gcloud/rest/datastore/key.py @@ -0,0 +1,86 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import + + +class PathElement(object): + def __init__(self, kind, id_=None, name=None): + # type: (str, Optional[int], Optional[str]) -> None + self.kind = kind + + self.id = id_ + self.name = name + if self.id and self.name: + raise Exception('invalid PathElement contains both ID and name') + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, PathElement): + return False + + return bool(self.kind == other.kind and self.id == other.id + and self.name == other.name) + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> PathElement + kind = data['kind'] # type: str + id_ = data.get('id') # type: Optional[int] + name = data.get('name') # type: Optional[str] + return cls(kind, id_=id_, name=name) + + def to_repr(self): + # type: () -> Dict[str, Any] + data = {'kind': self.kind} + if self.id: + data['id'] = self.id + elif self.name: + data['name'] = self.name + + return data + + +class Key(object): + path_element_kind = PathElement + + def __init__(self, project, path, namespace=''): + # type: (str, List[PathElement], str) -> None + self.project = project + self.namespace = namespace + self.path = path + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, Key): + return False + + return bool(self.project == other.project + and self.namespace == other.namespace + and self.path == other.path) + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> Key + return cls(data['partitionId']['projectId'], + path=[cls.path_element_kind.from_repr(p) + for p in data['path']], + namespace=data['partitionId'].get('namespaceId', '')) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'partitionId': { + 'projectId': self.project, + 'namespaceId': self.namespace, + }, + 'path': [p.to_repr() for p in self.path], + } diff --git a/datastore/gcloud/rest/datastore/property_order.py b/datastore/gcloud/rest/datastore/property_order.py new file mode 100644 index 0000000..53c72f5 --- /dev/null +++ b/datastore/gcloud/rest/datastore/property_order.py @@ -0,0 +1,39 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import + +from gcloud.rest.datastore.constants import Direction + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#PropertyOrder +class PropertyOrder(object): + def __init__(self, prop, direction=Direction.ASCENDING): + # type: (str, Direction) -> None + self.prop = prop + self.direction = direction + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, PropertyOrder): + return False + + return bool( + self.prop == other.prop + and self.direction == other.direction) + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> PropertyOrder + prop = data['property']['name'] + direction = Direction(data['direction']) + return cls(prop=prop, direction=direction) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'property': {'name': self.prop}, + 'direction': self.direction.value, + } diff --git a/datastore/gcloud/rest/datastore/query.py b/datastore/gcloud/rest/datastore/query.py new file mode 100644 index 0000000..3c53d8f --- /dev/null +++ b/datastore/gcloud/rest/datastore/query.py @@ -0,0 +1,221 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import + +from gcloud.rest.datastore.constants import MoreResultsType +from gcloud.rest.datastore.constants import ResultType +from gcloud.rest.datastore.entity import EntityResult +from gcloud.rest.datastore.filter import Filter +from gcloud.rest.datastore.property_order import PropertyOrder +from gcloud.rest.datastore.value import Value + + +class BaseQuery(object): + json_key = '' + value_kind = Value + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> BaseQuery + raise NotImplementedError + + def to_repr(self): + # type: () -> Dict[str, Any] + raise NotImplementedError + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#Query +class Query(BaseQuery): + json_key = 'query' + + # TODO: support `projection` and `distinctOn` + def __init__(self, + kind='', # type: str + query_filter=None, # type: Optional[Filter] + order=None, # type: Optional[List[PropertyOrder]] + start_cursor='', # type: str + end_cursor='', # type: str + offset=0, # type: int + limit=0 # type: int + ): + # type: (...) -> None + self.kind = kind + self.query_filter = query_filter + self.orders = order or [] + self.start_cursor = start_cursor + self.end_cursor = end_cursor + self.offset = offset + self.limit = limit + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, Query): + return False + + return bool( + self.kind == other.kind + and self.query_filter == other.query_filter) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> Query + kind = data['kind'] or '' # Kind is required + orders = [PropertyOrder.from_repr(o) for o in data.get('order', [])] + start_cursor = data.get('startCursor') or '' + end_cursor = data.get('endCursor') or '' + offset = int(data.get('offset') or 0) + limit = int(data.get('limit') or 0) + + filter_ = data.get('filter') + query_filter = Filter.from_repr(filter_) if filter_ else None + + return cls(kind=kind, query_filter=query_filter, order=orders, + start_cursor=start_cursor, end_cursor=end_cursor, + offset=offset, limit=limit) + + def to_repr(self): + # type: () -> Dict[str, Any] + data = {'kind': [{'name': self.kind}] if self.kind else []} + if self.query_filter: + data['filter'] = self.query_filter.to_repr() + if self.orders: + data['order'] = [o.to_repr() for o in self.orders] + if self.start_cursor: + data['startCursor'] = self.start_cursor + if self.end_cursor: + data['endCursor'] = self.end_cursor + if self.offset: + data['offset'] = self.offset + if self.limit: + data['limit'] = self.limit + return data + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#gqlquery +class GQLQuery(BaseQuery): + json_key = 'gqlQuery' + + def __init__(self, + query_string, # type: str + allow_literals=True, # type: bool + named_bindings=None, # type: Optional[Dict[str, Any]] + positional_bindings=None # type: Optional[List[Any]] + ): + # type: (...) -> None + self.query_string = query_string + self.allow_literals = allow_literals + self.named_bindings = named_bindings or {} + self.positional_bindings = positional_bindings or [] + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, GQLQuery): + return False + + return bool( + self.query_string == other.query_string + and self.allow_literals == other.allow_literals + and self.named_bindings == other.named_bindings + and self.positional_bindings == other.positional_bindings) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> GQLQuery + allow_literals = data['allowLiterals'] + query_string = data['queryString'] + named_bindings = {k: cls.value_kind.from_repr(v['value'].value) + for k, v in data.get('namedBindings', {}).items()} + positional_bindings = [cls.value_kind.from_repr(v['value'].value) + for v in data.get('positionalBindings', [])] + return cls(query_string, allow_literals=allow_literals, + named_bindings=named_bindings, + positional_bindings=positional_bindings) + + def to_repr(self): + # type: () -> Dict[str, Any] + return { + 'allowLiterals': self.allow_literals, + 'queryString': self.query_string, + 'namedBindings': {k: {'value': self.value_kind(v).to_repr()} + for k, v in self.named_bindings.items()}, + 'positionalBindings': [{'value': self.value_kind(v).to_repr()} + for v in self.positional_bindings], + } + + +class QueryResultBatch(object): + entity_result_kind = EntityResult + + def __init__(self, + end_cursor, # type: str + entity_result_type=ResultType.UNSPECIFIED, # type: ResultType + entity_results=None, # type: Optional[List[EntityResult]] + more_results=MoreResultsType.UNSPECIFIED, + skipped_cursor='', # type: str + skipped_results=0, # type: int + snapshot_version='' # type: str + ): + # type: (...) -> None + self.end_cursor = end_cursor + + self.entity_result_type = entity_result_type + self.entity_results = entity_results or [] + self.more_results = more_results + self.skipped_cursor = skipped_cursor + self.skipped_results = skipped_results + self.snapshot_version = snapshot_version + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, QueryResultBatch): + return False + + return bool(self.end_cursor == other.end_cursor + and self.entity_result_type == other.entity_result_type + and self.entity_results == other.entity_results + and self.more_results == other.more_results + and self.skipped_cursor == other.skipped_cursor + and self.skipped_results == other.skipped_results + and self.snapshot_version == other.snapshot_version) + + def __repr__(self): + # type: () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> QueryResultBatch + end_cursor = data['endCursor'] + entity_result_type = ResultType(data['entityResultType']) + entity_results = [cls.entity_result_kind.from_repr(er) + for er in data.get('entityResults', [])] + more_results = MoreResultsType(data['moreResults']) + skipped_cursor = data.get('skippedCursor', '') + skipped_results = data.get('skippedResults', 0) + snapshot_version = data.get('snapshotVersion', '') + return cls(end_cursor, entity_result_type=entity_result_type, + entity_results=entity_results, more_results=more_results, + skipped_cursor=skipped_cursor, + skipped_results=skipped_results, + snapshot_version=snapshot_version) + + def to_repr(self): + # type: () -> Dict[str, Any] + data = { + 'endCursor': self.end_cursor, + 'entityResults': [er.to_repr() for er in self.entity_results], + 'entityResultType': self.entity_result_type.value, + 'moreResults': self.more_results.value, + 'skippedResults': self.skipped_results, + } + if self.skipped_cursor: + data['skippedCursor'] = self.skipped_cursor + if self.snapshot_version: + data['snapshotVersion'] = self.snapshot_version + + return data diff --git a/datastore/gcloud/rest/datastore/value.py b/datastore/gcloud/rest/datastore/value.py new file mode 100644 index 0000000..acc6d8c --- /dev/null +++ b/datastore/gcloud/rest/datastore/value.py @@ -0,0 +1,94 @@ +from datetime import datetime +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import + +from gcloud.rest.datastore.constants import TypeName +from gcloud.rest.datastore.constants import TYPES +from gcloud.rest.datastore.key import Key + + +# https://cloud.google.com/datastore/docs/reference/data/rest/v1/projects/runQuery#value +class Value(object): + key_kind = Key + + def __init__(self, value, exclude_from_indexes=False): + # type: (Any, bool) -> None + self.value = value + self.excludeFromIndexes = exclude_from_indexes + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, Value): + return False + + return bool( + self.excludeFromIndexes == other.excludeFromIndexes + and self.value == other.value) + + def __repr__(self): + # type () -> str + return str(self.to_repr()) + + @classmethod + def from_repr(cls, data): + # type: (Dict[str, Any]) -> Value + supported_types = cls._get_supported_types() + for value_type, type_name in supported_types.items(): + json_key = type_name.value + if json_key in data: + if json_key == 'nullValue': + value = None + elif value_type == datetime: + value = datetime.strptime(data[json_key], + '%Y-%m-%dT%H:%M:%S.%f000Z') + elif value_type == cls.key_kind: + value = cls.key_kind.from_repr(data[json_key]) + else: + value = value_type(data[json_key]) + break + else: + supported = [name.value for name in supported_types.values()] + raise NotImplementedError( + '%s does not contain a supported value type' + ' (any of: %s)' % (data.keys(), supported)) + + # Google may not populate that field. This can happen with both + # indexed and non-indexed fields. + exclude_from_indexes = bool(data.get('excludeFromIndexes', False)) + + return cls(value=value, exclude_from_indexes=exclude_from_indexes) + + def to_repr(self): + # type: () -> Dict[str, Any] + value_type = self._infer_type(self.value) + if value_type == TypeName.KEY: + value = self.value.to_repr() + elif value_type == TypeName.TIMESTAMP: + value = self.value.strftime('%Y-%m-%dT%H:%M:%S.%f000Z') + else: + value = 'NULL_VALUE' if self.value is None else self.value + return { + 'excludeFromIndexes': self.excludeFromIndexes, + value_type.value: value, + } + + def _infer_type(self, value): + # type: (Any) -> TypeName + kind = type(value) + supported_types = self._get_supported_types() + + try: + return supported_types[kind] + except KeyError: + raise NotImplementedError( + '%s is not a supported value type' + ' (any of: %s)' % (kind, supported_types)) + + @classmethod + def _get_supported_types(cls): + # type: () -> Dict[type, TypeName] + supported_types = TYPES + supported_types.update({ + cls.key_kind: TypeName.KEY, + }) + return supported_types diff --git a/datastore/noxfile.py b/datastore/noxfile.py new file mode 100644 index 0000000..983cbaf --- /dev/null +++ b/datastore/noxfile.py @@ -0,0 +1,41 @@ +import os + +import nox + + +@nox.session(python=['2.7', '3.5', '3.6', '3.7'], reuse_venv=True) +def unit_tests(session): + session.install('pytest', 'pytest-cov') + session.install('-e', '.') + + session.run('py.test', '--quiet', '--cov=gcloud.rest', '--cov=tests.unit', + '--cov-append', '--cov-report=', os.path.join('tests', 'unit'), + *session.posargs) + + +@nox.session(python=['2.7', '3.7'], reuse_venv=True) +def integration_tests(session): + if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS'): + session.skip('Credentials must be set via environment variable.') + + session.install('pytest', 'pytest-cov', 'pytest-mock') + session.install('.') + + session.run('py.test', '--quiet', '--cov=gcloud.rest', + '--cov=tests.integration', '--cov-append', '--cov-report=', + os.path.join('tests', 'integration'), *session.posargs) + + +@nox.session(python=['2.7', '3.7'], reuse_venv=True) +def lint_setup_py(session): + session.install('docutils', 'Pygments') + session.run('python', 'setup.py', 'check', '--restructuredtext', + '--strict') + + +@nox.session(python=['3.7'], reuse_venv=True) +def cover(session): + session.install('coverage', 'pytest-cov') + + session.run('coverage', 'report', '--show-missing') + session.run('coverage', 'erase') diff --git a/datastore/requirements.txt b/datastore/requirements.txt new file mode 100644 index 0000000..ec5d5f4 --- /dev/null +++ b/datastore/requirements.txt @@ -0,0 +1,3 @@ +gcloud-rest >= 1.0.0, < 2.0.0 +requests[security] >= 2.0.0, < 3.0.0 +typing >= 3.0.0, < 4.0.0 diff --git a/datastore/setup.cfg b/datastore/setup.cfg new file mode 100644 index 0000000..2a9acf1 --- /dev/null +++ b/datastore/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/datastore/setup.py b/datastore/setup.py new file mode 100644 index 0000000..e228400 --- /dev/null +++ b/datastore/setup.py @@ -0,0 +1,45 @@ +import os + +import setuptools + + +PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as f: + README = f.read() + +with open(os.path.join(PACKAGE_ROOT, 'requirements.txt')) as f: + REQUIREMENTS = [r.strip() for r in f.readlines()] + + +setuptools.setup( + name='gcloud-rest-datastore', + version='1.0.0', + description='RESTful Python Client for Google Cloud Datastore', + long_description=README, + namespace_packages=[ + 'gcloud', + 'gcloud.rest', + ], + packages=setuptools.find_packages(exclude=('tests',)), + install_requires=REQUIREMENTS, + author='TalkIQ', + author_email='engineering@talkiq.com', + url='https://github.com/talkiq/gcloud-rest', + platforms='Posix; MacOS X; Windows', + include_package_data=True, + zip_safe=False, + license='MIT License', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Topic :: Internet', + ], +) diff --git a/datastore/tests/__init__.py b/datastore/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datastore/tests/integration/__init__.py b/datastore/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datastore/tests/integration/conftest.py b/datastore/tests/integration/conftest.py new file mode 100644 index 0000000..221d9db --- /dev/null +++ b/datastore/tests/integration/conftest.py @@ -0,0 +1,28 @@ +import os + +import pytest + + +@pytest.fixture(scope='module') # type: ignore +def creds(): + # type: () -> str + # TODO: bundle public creds into this repo + return os.environ['GOOGLE_APPLICATION_CREDENTIALS'] + + +@pytest.fixture(scope='module') # type: ignore +def kind(): + # type: () -> str + return 'public_test' + + +@pytest.fixture(scope='module') # type: ignore +def project(): + # type: () -> str + return 'voiceai-staging' + + +@pytest.fixture(scope='module') # type: ignore +def export_bucket_name(): + # type: () -> str + return 'voiceai-staging-public-test' diff --git a/datastore/tests/integration/query_test.py b/datastore/tests/integration/query_test.py new file mode 100644 index 0000000..5bd2665 --- /dev/null +++ b/datastore/tests/integration/query_test.py @@ -0,0 +1,69 @@ +import requests + +from gcloud.rest.datastore import Datastore +from gcloud.rest.datastore import Filter +from gcloud.rest.datastore import GQLQuery +from gcloud.rest.datastore import Key +from gcloud.rest.datastore import Operation +from gcloud.rest.datastore import PathElement +from gcloud.rest.datastore import PropertyFilter +from gcloud.rest.datastore import PropertyFilterOperator +from gcloud.rest.datastore import Query +from gcloud.rest.datastore import Value + + +def test_query(creds, kind, project): + # type: (str, str, str) -> None + with requests.Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + property_filter = PropertyFilter( + prop='value', operator=PropertyFilterOperator.EQUAL, + value=Value(42)) + query = Query(kind=kind, query_filter=Filter(property_filter)) + + before = ds.runQuery(query, session=s) + num_results = len(before.entity_results) + + transaction = ds.beginTransaction(session=s) + mutations = [ + ds.make_mutation(Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 42}), + ds.make_mutation(Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 42}), + ] + ds.commit(mutations, transaction=transaction, session=s) + + after = ds.runQuery(query, session=s) + assert len(after.entity_results) == num_results + 2 + + +def test_gql_query(creds, kind, project): + # type: (str, str, str) -> None + with requests.Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + query = GQLQuery('SELECT * FROM %s WHERE value = @value' % kind, + named_bindings={'value': 42}) + + before = ds.runQuery(query, session=s) + num_results = len(before.entity_results) + + transaction = ds.beginTransaction(session=s) + mutations = [ + ds.make_mutation(Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 42}), + ds.make_mutation(Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 42}), + ds.make_mutation(Operation.INSERT, + Key(project, [PathElement(kind)]), + properties={'value': 42}), + ] + ds.commit(mutations, transaction=transaction, session=s) + + after = ds.runQuery(query, session=s) + assert len(after.entity_results) == num_results + 3 diff --git a/datastore/tests/integration/smoke_test.py b/datastore/tests/integration/smoke_test.py new file mode 100644 index 0000000..0f962ee --- /dev/null +++ b/datastore/tests/integration/smoke_test.py @@ -0,0 +1,75 @@ +import uuid + +import requests + +from gcloud.rest.datastore import Datastore +from gcloud.rest.datastore import Key +from gcloud.rest.datastore import Operation +from gcloud.rest.datastore import PathElement + + +def test_item_lifecycle(creds, kind, project): + # type: (str, str, str) -> None + key = Key(project, [PathElement(kind)]) + + with requests.Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + allocatedKeys = ds.allocateIds([key], session=s) + assert len(allocatedKeys) == 1 + key.path[-1].id = allocatedKeys[0].path[-1].id + assert key == allocatedKeys[0] + + ds.reserveIds(allocatedKeys, session=s) + + props_insert = {'is_this_bad_data': True} + ds.insert(allocatedKeys[0], props_insert, session=s) + actual = ds.lookup([allocatedKeys[0]], session=s) + assert actual['found'][0].entity.properties == props_insert + + props_update = {'animal': 'aardvark', 'overwrote_bad_data': True} + ds.update(allocatedKeys[0], props_update, session=s) + actual = ds.lookup([allocatedKeys[0]], session=s) + assert actual['found'][0].entity.properties == props_update + + props_upsert = {'meaning_of_life': 42} + ds.upsert(allocatedKeys[0], props_upsert, session=s) + actual = ds.lookup([allocatedKeys[0]], session=s) + assert actual['found'][0].entity.properties == props_upsert + + ds.delete(allocatedKeys[0], session=s) + actual = ds.lookup([allocatedKeys[0]], session=s) + assert len(actual['missing']) == 1 + + +def test_transaction(creds, kind, project): + # type: (str, str, str) -> None + path_element_name = 'test_record_%s' % uuid.uuid4() + key = Key(project, [PathElement(kind, name=path_element_name)]) + + with requests.Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + transaction = ds.beginTransaction(session=s) + actual = ds.lookup([key], transaction=transaction, session=s) + assert len(actual['missing']) == 1 + + mutations = [ + ds.make_mutation(Operation.INSERT, key, + properties={'animal': 'three-toed sloth'}), + ds.make_mutation(Operation.UPDATE, key, + properties={'animal': 'aardvark'}), + ] + ds.commit(mutations, transaction=transaction, session=s) + + actual = ds.lookup([key], session=s) + assert actual['found'][0].entity.properties == {'animal': 'aardvark'} + + +def test_rollback(creds, project): + # type: (str, str) -> None + with requests.Session() as s: + ds = Datastore(project=project, service_file=creds, session=s) + + transaction = ds.beginTransaction(session=s) + ds.rollback(transaction, session=s) diff --git a/datastore/tests/unit/__init__.py b/datastore/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datastore/tests/unit/constants_test.py b/datastore/tests/unit/constants_test.py new file mode 100644 index 0000000..8c6ea95 --- /dev/null +++ b/datastore/tests/unit/constants_test.py @@ -0,0 +1,5 @@ +import gcloud.rest.datastore.constants as constants # pylint: disable=unused-import + + +def test_importable(): + assert True diff --git a/datastore/tests/unit/datastore_test.py b/datastore/tests/unit/datastore_test.py new file mode 100644 index 0000000..220db8a --- /dev/null +++ b/datastore/tests/unit/datastore_test.py @@ -0,0 +1,5 @@ +import gcloud.rest.datastore.datastore as datastore # pylint: disable=unused-import + + +def test_importable(): + assert True diff --git a/datastore/tests/unit/filter_test.py b/datastore/tests/unit/filter_test.py new file mode 100644 index 0000000..e2007c2 --- /dev/null +++ b/datastore/tests/unit/filter_test.py @@ -0,0 +1,180 @@ +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import + +import pytest + +from gcloud.rest.datastore import CompositeFilter +from gcloud.rest.datastore import CompositeFilterOperator +from gcloud.rest.datastore import Filter +from gcloud.rest.datastore import PropertyFilter +from gcloud.rest.datastore import PropertyFilterOperator +from gcloud.rest.datastore import Value + + +class TestFilter(object): # pylint: disable=no-init + @staticmethod + def test_property_filter_from_repr(property_filters): + original_filter = property_filters[0] + data = { + 'property': { + 'name': original_filter.prop + }, + 'op': original_filter.operator, + 'value': original_filter.value.to_repr() + } + + output_filter = PropertyFilter.from_repr(data) + + assert output_filter == original_filter + + def test_property_filter_to_repr(self, property_filters): + property_filter = property_filters[0] + query_filter = Filter(inner_filter=property_filter) + + r = query_filter.to_repr() + + self._assert_is_correct_prop_dict_for_property_filter( + r['propertyFilter'], property_filter) + + @staticmethod + def test_composite_filter_from_repr(property_filters): + original_filter = CompositeFilter( + operator=CompositeFilterOperator.AND, + filters=[ + Filter(property_filters[0]), + Filter(property_filters[1]) + ]) + data = { + 'op': original_filter.operator, + 'filters': [ + { + 'propertyFilter': { + 'property': { + 'name': + original_filter.filters[0].inner_filter.prop + }, + 'op': original_filter.filters[0].inner_filter.operator, + 'value': property_filters[0].value.to_repr() + } + }, + { + 'propertyFilter': { + 'property': { + 'name': + original_filter.filters[1].inner_filter.prop + }, + 'op': original_filter.filters[1].inner_filter.operator, + 'value': property_filters[1].value.to_repr() + } + }, + ] + } + + output_filter = CompositeFilter.from_repr(data) + + assert output_filter == original_filter + + def test_composite_filter_to_repr(self, property_filters): + composite_filter = CompositeFilter( + operator=CompositeFilterOperator.AND, + filters=[ + Filter(property_filters[0]), + Filter(property_filters[1]) + ]) + query_filter = Filter(composite_filter) + + r = query_filter.to_repr() + + composite_filter_dict = r['compositeFilter'] + assert composite_filter_dict['op'] == 'AND' + self._assert_is_correct_prop_dict_for_property_filter( + composite_filter_dict['filters'][0]['propertyFilter'], + property_filters[0]) + self._assert_is_correct_prop_dict_for_property_filter( + composite_filter_dict['filters'][1]['propertyFilter'], + property_filters[1]) + + @staticmethod + def test_filter_from_repr(composite_filter): + original_filter = Filter(inner_filter=composite_filter) + + data = { + 'compositeFilter': original_filter.inner_filter.to_repr() + } + + output_filter = Filter.from_repr(data) + + assert output_filter == original_filter + + @staticmethod + def test_filter_from_repr_unexpected_filter_name(): + unexpected_filter_name = 'unexpectedFilterName' + data = { + unexpected_filter_name: 'DoesNotMatter' + } + + with pytest.raises(ValueError) as ex_info: + Filter.from_repr(data) + + assert unexpected_filter_name in ex_info.value.args[0] + + @staticmethod + def test_filter_to_repr(composite_filter): + test_filter = Filter(inner_filter=composite_filter) + + r = test_filter.to_repr() + + assert r['compositeFilter'] == test_filter.inner_filter.to_repr() + + @staticmethod + def test_repr_returns_to_repr_as_string(query_filter): + assert repr(query_filter) == str(query_filter.to_repr()) + + @staticmethod + @pytest.fixture() + def property_filters(): + # type: () -> List[PropertyFilter] + return [ + PropertyFilter( + prop='prop1', + operator=PropertyFilterOperator.LESS_THAN, + value=Value('value1') + ), + PropertyFilter( + prop='prop2', + operator=PropertyFilterOperator.GREATER_THAN, + value=Value(1234) + ) + ] + + @staticmethod + @pytest.fixture() + def composite_filter(property_filters): + # type: (List[PropertyFilter]) -> CompositeFilter + return CompositeFilter( + operator=CompositeFilterOperator.AND, + filters=[ + Filter(property_filters[0]), + Filter(property_filters[1]) + ]) + + @staticmethod + @pytest.fixture() + def query_filter(composite_filter): + # type: (CompositeFilter) -> Filter + return Filter(inner_filter=composite_filter) + + @staticmethod + @pytest.fixture() + def value(): + # type: () -> Value + return Value('value') + + @staticmethod + def _assert_is_correct_prop_dict_for_property_filter( + prop_dict, property_filter): + # type: (Dict[str, Any], PropertyFilter) -> None + assert prop_dict['property']['name'] == property_filter.prop + assert prop_dict['op'] == property_filter.operator.value + assert prop_dict['value'] == property_filter.value.to_repr() diff --git a/datastore/tests/unit/property_order_test.py b/datastore/tests/unit/property_order_test.py new file mode 100644 index 0000000..2cc0e58 --- /dev/null +++ b/datastore/tests/unit/property_order_test.py @@ -0,0 +1,45 @@ +import pytest + +from gcloud.rest.datastore import Direction +from gcloud.rest.datastore import PropertyOrder + + +class TestPropertyOrder(object): # pylint: disable=no-init + @staticmethod + def test_order_defaults_to_ascending(): + assert PropertyOrder('prop_name').direction == Direction.ASCENDING + + @staticmethod + def test_order_from_repr(property_order): + original_order = property_order + data = { + 'property': { + 'name': original_order.prop + }, + 'direction': original_order.direction + } + + output_order = PropertyOrder.from_repr(data) + + assert output_order == original_order + + @staticmethod + def test_order_to_repr(): + property_name = 'my_prop' + direction = Direction.DESCENDING + order = PropertyOrder(property_name, direction) + + r = order.to_repr() + + assert r['property']['name'] == property_name + assert r['direction'] == direction.value + + @staticmethod + def test_repr_returns_to_repr_as_string(property_order): + assert repr(property_order) == str(property_order.to_repr()) + + @staticmethod + @pytest.fixture() + def property_order(): + # type: () -> PropertyOrder + return PropertyOrder(prop='prop_name', direction=Direction.DESCENDING) diff --git a/datastore/tests/unit/query_test.py b/datastore/tests/unit/query_test.py new file mode 100644 index 0000000..3522c23 --- /dev/null +++ b/datastore/tests/unit/query_test.py @@ -0,0 +1,126 @@ +import pytest + +from gcloud.rest.datastore import Direction +from gcloud.rest.datastore import Filter +from gcloud.rest.datastore import PropertyFilter +from gcloud.rest.datastore import PropertyFilterOperator +from gcloud.rest.datastore import PropertyOrder +from gcloud.rest.datastore import Query +from gcloud.rest.datastore import Value + + +class TestQuery(object): # pylint: disable=no-init + @staticmethod + def test_from_repr(query): + original_query = query + data = { + 'kind': original_query.kind, + 'filter': original_query.query_filter.to_repr() + } + + output_query = Query.from_repr(data) + + assert output_query == original_query + + @staticmethod + def test_from_repr_query_without_kind(query_filter): + original_query = Query(kind='', query_filter=query_filter) + data = { + 'kind': [], + 'filter': original_query.query_filter.to_repr() + } + + output_query = Query.from_repr(data) + + assert output_query == original_query + + @staticmethod + def test_from_repr_query_with_several_orders(): + orders = [ + PropertyOrder('property1', direction=Direction.ASCENDING), + PropertyOrder('property2', direction=Direction.DESCENDING) + ] + original_query = Query(order=orders) + + data = { + 'kind': [], + 'order': [ + { + 'property': { + 'name': orders[0].prop + }, + 'direction': orders[0].direction + }, + { + 'property': { + 'name': orders[1].prop + }, + 'direction': orders[1].direction + } + ] + } + + output_query = Query.from_repr(data) + + assert output_query == original_query + + @staticmethod + def test_to_repr_simple_query(): + kind = 'foo' + query = Query(kind) + + r = query.to_repr() + + assert len(r['kind']) == 1 + assert r['kind'][0]['name'] == kind + + @staticmethod + def test_to_repr_query_without_kind(): + query = Query() + + r = query.to_repr() + + assert not r['kind'] + + @staticmethod + def test_to_repr_query_with_filter(query_filter): + property_filter = query_filter + query = Query('foo', property_filter) + + r = query.to_repr() + + assert r['filter'] == property_filter.to_repr() + + @staticmethod + def test_to_repr_query_with_several_orders(): + orders = [ + PropertyOrder('property1', direction=Direction.ASCENDING), + PropertyOrder('property2', direction=Direction.DESCENDING) + ] + query = Query(order=orders) + + r = query.to_repr() + + assert len(r['order']) == 2 + assert r['order'][0] == orders[0].to_repr() + assert r['order'][1] == orders[1].to_repr() + + @staticmethod + def test_repr_returns_to_repr_as_string(query): + assert repr(query) == str(query.to_repr()) + + @staticmethod + @pytest.fixture() + def query(query_filter): + # type: (Filter) -> Query + return Query('query_kind', query_filter) + + @staticmethod + @pytest.fixture() + def query_filter(): + # type: () -> Filter + inner_filter = PropertyFilter( + prop='property_name', + operator=PropertyFilterOperator.EQUAL, + value=Value(123)) + return Filter(inner_filter) diff --git a/datastore/tests/unit/value_test.py b/datastore/tests/unit/value_test.py new file mode 100644 index 0000000..76f2c22 --- /dev/null +++ b/datastore/tests/unit/value_test.py @@ -0,0 +1,188 @@ +from datetime import datetime + +import pytest + +from gcloud.rest.datastore import Key +from gcloud.rest.datastore import PathElement +from gcloud.rest.datastore import Value + + +class TestValue(object): + @staticmethod + @pytest.mark.parametrize('json_key,json_value', [ + # TODO Make this Python 2 compatible + # https://docs.python.org/3/whatsnew/2.6.html#pep-3112-byte-literals + # ('blobValue', bytes('foobar', 'utf-8')), + ('booleanValue', True), + ('doubleValue', 34.48), + ('integerValue', 8483), + ('stringValue', 'foobar'), + # TODO Make this Python 2 compatible + # ('blobValue', b''), + ('booleanValue', False), + ('doubleValue', 0.0), + ('integerValue', 0), + ('stringValue', ''), + ]) + def test_from_repr(json_key, json_value): + data = { + 'excludeFromIndexes': False, + json_key: json_value + } + + value = Value.from_repr(data) + + assert value.excludeFromIndexes is False + assert value.value == json_value + + @staticmethod + def test_from_repr_with_null_value(): + data = { + 'excludeFromIndexes': False, + 'nullValue': 'NULL_VALUE' + } + + value = Value.from_repr(data) + + assert value.excludeFromIndexes is False + assert value.value is None + + @staticmethod + def test_from_repr_with_datetime_value(): + data = { + 'excludeFromIndexes': False, + 'timestampValue': '1998-07-12T11:22:33.456789000Z' + } + + value = Value.from_repr(data) + + expected_value = datetime(year=1998, month=7, day=12, hour=11, + minute=22, second=33, microsecond=456789) + assert value.value == expected_value + + @staticmethod + def test_from_repr_with_key_value(key): + data = { + 'excludeFromIndexes': False, + 'keyValue': key.to_repr() + } + + value = Value.from_repr(data) + + assert value.value == key + + @staticmethod + def test_from_repr_could_not_find_supported_value_key(): + data = { + 'excludeFromIndexes': False, + } + + with pytest.raises(NotImplementedError) as ex_info: + Value.from_repr(data) + + assert 'excludeFromIndexes' in ex_info.value.args[0] + + @staticmethod + @pytest.mark.parametrize('v,expected_json_key', [ + # TODO Make this Python 2 compatible + # https://docs.python.org/3/whatsnew/2.6.html#pep-3112-byte-literals + # (bytes('foobar', 'utf-8'), 'blobValue'), + (True, 'booleanValue'), + (34.48, 'doubleValue'), + (8483, 'integerValue'), + ('foobar', 'stringValue'), + # TODO Make this Python 2 compatible + # (b'', 'blobValue'), + (False, 'booleanValue'), + (0.0, 'doubleValue'), + (0, 'integerValue'), + ('', 'stringValue'), + ]) + def test_to_repr(v, expected_json_key): + value = Value(v) + + r = value.to_repr() + + assert len(r) == 2 # Value + excludeFromIndexes + assert r['excludeFromIndexes'] is False + assert r[expected_json_key] == v + + @staticmethod + def test_to_repr_with_null_value(): + value = Value(None) + + r = value.to_repr() + + assert r['nullValue'] == 'NULL_VALUE' + + @staticmethod + def test_to_repr_with_datetime_value(): + dt = datetime(year=2018, month=7, day=15, hour=11, minute=22, + second=33, microsecond=456789) + value = Value(dt) + + r = value.to_repr() + + assert r['timestampValue'] == '2018-07-15T11:22:33.456789000Z' + + @staticmethod + def test_to_repr_with_key_value(key): + value = Value(key) + + r = value.to_repr() + + assert r['keyValue'] == key.to_repr() + + @staticmethod + def test_to_repr_with_custom_key_value(): + class CustomKey(object): + @staticmethod + def to_repr(): + return {'some': 'JSON'} + + class CustomValue(Value): + key_kind = CustomKey + + key = CustomKey() + value = CustomValue(key) + + r = value.to_repr() + + assert r['keyValue'] == key.to_repr() + + @staticmethod + def test_to_repr_exclude_from_indexes(): + value = Value(123, exclude_from_indexes=True) + + r = value.to_repr() + + assert r['excludeFromIndexes'] + + @staticmethod + def test_to_repr_non_supported_type(): + class NonSupportedType(object): + pass + value = Value(NonSupportedType()) + + with pytest.raises(Exception) as ex_info: + value.to_repr() + + assert NonSupportedType.__name__ in ex_info.value.args[0] + + @staticmethod + def test_repr_returns_to_repr_as_string(value): + assert repr(value) == str(value.to_repr()) + + @staticmethod + @pytest.fixture() + def key(): + # type: () -> Key + path = PathElement(kind='my-kind', name='path-name') + key = Key(project='my-project', path=[path], namespace='my-namespace') + return key + + @staticmethod + @pytest.fixture() + def value(): + # type: () -> Value + return Value(value='foobar', exclude_from_indexes=False)