From eba6a77ab29ca1eef19a1c869495718bbc13ade3 Mon Sep 17 00:00:00 2001 From: Gabriela Surita Date: Wed, 1 Feb 2017 19:35:54 -0200 Subject: [PATCH 1/4] Move generic schemas outside resource This is needed to allow importing generic schemas on other parts of kinto core without breaking the import order. Related to #1033 --- kinto/core/resource/schema.py | 114 +---------------------------- kinto/core/schema.py | 110 ++++++++++++++++++++++++++++ tests/core/resource/test_schema.py | 22 ------ tests/core/test_schema.py | 26 +++++++ 4 files changed, 138 insertions(+), 134 deletions(-) create mode 100644 kinto/core/schema.py create mode 100644 tests/core/test_schema.py diff --git a/kinto/core/resource/schema.py b/kinto/core/resource/schema.py index 88335b264..cdc52b73c 100644 --- a/kinto/core/resource/schema.py +++ b/kinto/core/resource/schema.py @@ -1,8 +1,7 @@ -import six import colander -from colander import SchemaNode, String -from kinto.core.utils import strip_whitespace, msec_time, decode_header, native_value +from kinto.core.schema import Any, HeaderField, QueryField, HeaderQuotedInteger, FieldList +from kinto.core.utils import native_value # Resource related schemas @@ -117,115 +116,6 @@ def _get_node_principals(self, perm): missing=colander.drop) -# Generic schema nodes - - -class TimeStamp(colander.SchemaNode): - """Basic integer schema field that can be set to current server timestamp - in milliseconds if no value is provided. - - .. code-block:: python - - class Book(ResourceSchema): - added_on = TimeStamp() - read_on = TimeStamp(auto_now=False, missing=-1) - """ - schema_type = colander.Integer - - title = 'Epoch timestamp' - """Default field title.""" - - auto_now = True - """Set to current server timestamp (*milliseconds*) if not provided.""" - - missing = None - """Default field value if not provided in record.""" - - def deserialize(self, cstruct=colander.null): - if cstruct is colander.null and self.auto_now: - cstruct = msec_time() - return super(TimeStamp, self).deserialize(cstruct) - - -class URL(SchemaNode): - """String field representing a URL, with max length of 2048. - This is basically a shortcut for string field with - `~colander:colander.url`. - - .. code-block:: python - - class BookmarkSchema(ResourceSchema): - url = URL() - """ - schema_type = String - validator = colander.All(colander.url, colander.Length(min=1, max=2048)) - - def preparer(self, appstruct): - return strip_whitespace(appstruct) - - -class Any(colander.SchemaType): - """Colander type agnostic field.""" - - def deserialize(self, node, cstruct): - return cstruct - - -class HeaderField(colander.SchemaNode): - """Basic header field SchemaNode.""" - - missing = colander.drop - - def deserialize(self, cstruct=colander.null): - if isinstance(cstruct, six.binary_type): - try: - cstruct = decode_header(cstruct) - except UnicodeDecodeError: - raise colander.Invalid(self, msg='Headers should be UTF-8 encoded') - return super(HeaderField, self).deserialize(cstruct) - - -class QueryField(colander.SchemaNode): - """Basic querystring field SchemaNode.""" - - missing = colander.drop - - def deserialize(self, cstruct=colander.null): - if isinstance(cstruct, six.string_types): - cstruct = native_value(cstruct) - return super(QueryField, self).deserialize(cstruct) - - -class FieldList(QueryField): - """String field representing a list of attributes.""" - - schema_type = colander.Sequence - error_message = "The value should be a list of comma separated attributes" - missing = colander.drop - fields = colander.SchemaNode(colander.String(), missing=colander.drop) - - def deserialize(self, cstruct=colander.null): - if isinstance(cstruct, six.string_types): - cstruct = cstruct.split(',') - return super(FieldList, self).deserialize(cstruct) - - -class HeaderQuotedInteger(HeaderField): - """Integer between "" used in precondition headers.""" - - schema_type = colander.String - error_message = "The value should be integer between double quotes" - validator = colander.Any(colander.Regex('^"([0-9]+?)"$', msg=error_message), - colander.Regex('\*')) - - def deserialize(self, cstruct=colander.null): - param = super(HeaderQuotedInteger, self).deserialize(cstruct) - if param is colander.drop or param == '*': - return param - - return int(param[1:-1]) - - # Header schemas diff --git a/kinto/core/schema.py b/kinto/core/schema.py new file mode 100644 index 000000000..9e8d4e215 --- /dev/null +++ b/kinto/core/schema.py @@ -0,0 +1,110 @@ +import six +import colander + +from kinto.core.utils import strip_whitespace, msec_time, decode_header, native_value + + +class TimeStamp(colander.SchemaNode): + """Basic integer schema field that can be set to current server timestamp + in milliseconds if no value is provided. + + .. code-block:: python + + class Book(ResourceSchema): + added_on = TimeStamp() + read_on = TimeStamp(auto_now=False, missing=-1) + """ + schema_type = colander.Integer + + title = 'Epoch timestamp' + """Default field title.""" + + auto_now = True + """Set to current server timestamp (*milliseconds*) if not provided.""" + + missing = None + """Default field value if not provided in record.""" + + def deserialize(self, cstruct=colander.null): + if cstruct is colander.null and self.auto_now: + cstruct = msec_time() + return super(TimeStamp, self).deserialize(cstruct) + + +class URL(colander.SchemaNode): + """String field representing a URL, with max length of 2048. + This is basically a shortcut for string field with + `~colander:colander.url`. + + .. code-block:: python + + class BookmarkSchema(ResourceSchema): + url = URL() + """ + schema_type = colander.String + validator = colander.All(colander.url, colander.Length(min=1, max=2048)) + + def preparer(self, appstruct): + return strip_whitespace(appstruct) + + +class Any(colander.SchemaType): + """Colander type agnostic field.""" + + def deserialize(self, node, cstruct): + return cstruct + + +class HeaderField(colander.SchemaNode): + """Basic header field SchemaNode.""" + + missing = colander.drop + + def deserialize(self, cstruct=colander.null): + if isinstance(cstruct, six.binary_type): + try: + cstruct = decode_header(cstruct) + except UnicodeDecodeError: + raise colander.Invalid(self, msg='Headers should be UTF-8 encoded') + return super(HeaderField, self).deserialize(cstruct) + + +class QueryField(colander.SchemaNode): + """Basic querystring field SchemaNode.""" + + missing = colander.drop + + def deserialize(self, cstruct=colander.null): + if isinstance(cstruct, six.string_types): + cstruct = native_value(cstruct) + return super(QueryField, self).deserialize(cstruct) + + +class FieldList(QueryField): + """String field representing a list of attributes.""" + + schema_type = colander.Sequence + error_message = "The value should be a list of comma separated attributes" + missing = colander.drop + fields = colander.SchemaNode(colander.String(), missing=colander.drop) + + def deserialize(self, cstruct=colander.null): + if isinstance(cstruct, six.string_types): + cstruct = cstruct.split(',') + return super(FieldList, self).deserialize(cstruct) + + +class HeaderQuotedInteger(HeaderField): + """Integer between "" used in precondition headers.""" + + schema_type = colander.String + error_message = "The value should be integer between double quotes" + validator = colander.Any(colander.Regex('^"([0-9]+?)"$', msg=error_message), + colander.Regex('\*')) + + def deserialize(self, cstruct=colander.null): + param = super(HeaderQuotedInteger, self).deserialize(cstruct) + if param is colander.drop or param == '*': + return param + + return int(param[1:-1]) diff --git a/tests/core/resource/test_schema.py b/tests/core/resource/test_schema.py index b833ad3af..0ace7da60 100644 --- a/tests/core/resource/test_schema.py +++ b/tests/core/resource/test_schema.py @@ -1,32 +1,10 @@ import six import colander -import mock from kinto.core.testing import unittest from kinto.core.resource import schema -class TimeStampTest(unittest.TestCase): - @mock.patch('kinto.core.resource.schema.msec_time') - def test_default_value_comes_from_timestamper(self, time_mocked): - time_mocked.return_value = 666 - default = schema.TimeStamp().deserialize(colander.null) - self.assertEqual(default, 666) - - -class URLTest(unittest.TestCase): - def test_supports_full_url(self): - url = 'https://user:pass@myserver:9999/feeling.html#anchor' - deserialized = schema.URL().deserialize(url) - self.assertEqual(deserialized, url) - - def test_raises_invalid_if_no_scheme(self): - url = 'myserver/feeling.html#anchor' - self.assertRaises(colander.Invalid, - schema.URL().deserialize, - url) - - class ResourceSchemaTest(unittest.TestCase): def test_preserves_unknown_fields_when_specified(self): diff --git a/tests/core/test_schema.py b/tests/core/test_schema.py new file mode 100644 index 000000000..1e1f96a14 --- /dev/null +++ b/tests/core/test_schema.py @@ -0,0 +1,26 @@ +import colander +import mock + +from kinto.core.testing import unittest +from kinto.core import schema + + +class TimeStampTest(unittest.TestCase): + @mock.patch('kinto.core.schema.msec_time') + def test_default_value_comes_from_timestamper(self, time_mocked): + time_mocked.return_value = 666 + default = schema.TimeStamp().deserialize(colander.null) + self.assertEqual(default, 666) + + +class URLTest(unittest.TestCase): + def test_supports_full_url(self): + url = 'https://user:pass@myserver:9999/feeling.html#anchor' + deserialized = schema.URL().deserialize(url) + self.assertEqual(deserialized, url) + + def test_raises_invalid_if_no_scheme(self): + url = 'myserver/feeling.html#anchor' + self.assertRaises(colander.Invalid, + schema.URL().deserialize, + url) From ea52f5241fc796572cb8be750f877f8d56488748 Mon Sep 17 00:00:00 2001 From: Gabriela Surita Date: Wed, 1 Feb 2017 19:42:36 -0200 Subject: [PATCH 2/4] Update and Fix CHANGELOG 5.3.2 is already released and some merges were listed on the wrong version. --- CHANGELOG.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7f5f5b7c6..cfe563033 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,8 +10,17 @@ This document describes changes between each past release. - Remove pypy supports. (#1049) +**Internal changes** + +- Permission schema children fields are now set during initialization instead of on + deserialization (#1046). +- Request schemas (including validation and deserialization) are now isolated by method + and endpoint type (#1047). +- Move generic API schemas (e.g TimeStamps and HeaderFields) from `kinto.core.resource.schema` + to a sepate file on `kinto.core.schema`. -5.3.2 (unreleased) + +5.3.2 (2017-01-31) ------------------ **Bug fixes** @@ -30,12 +39,8 @@ This document describes changes between each past release. - Remove JSON Patch content-type from accepted types on the viewset, since it is handled in a separate view (#1031). -- Permission schema children fields are now set during initialization instead of on - deserialization (#1046). - Upgraded to Kinto-Admin 1.8.1 - Configure the Kinto Admin auth methods from the server configuration (#1042) -- Request schemas (including validation and deserialization) are now isolated by method - and endpoint type (#1047). 5.3.0 (2017-01-20) ------------------ From 40dd5b6b9510434fbfa18c808b5a2cb5bb631fc4 Mon Sep 17 00:00:00 2001 From: Gabriela Surita Date: Fri, 3 Feb 2017 00:04:11 -0200 Subject: [PATCH 3/4] deprecation and docstring comments Keep backward compatibility from all the schemas moved from `kinto.core.resource.schema` to `kinto.core.schema` by keeping a deprecated reference to the missing schemas (URL and TimeStamp). Also set a docstring on `kinto.core.schema` saying which schemas should be included there and instructions to where other schemas should live. --- kinto/core/resource/schema.py | 25 ++++++++++++++++++++++++- kinto/core/schema.py | 10 ++++++++++ tests/core/resource/test_schema.py | 18 ++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/kinto/core/resource/schema.py b/kinto/core/resource/schema.py index cdc52b73c..666d60c60 100644 --- a/kinto/core/resource/schema.py +++ b/kinto/core/resource/schema.py @@ -1,9 +1,32 @@ +import warnings + import colander -from kinto.core.schema import Any, HeaderField, QueryField, HeaderQuotedInteger, FieldList +from kinto.core.schema import (Any, HeaderField, QueryField, HeaderQuotedInteger, + FieldList, TimeStamp, URL) from kinto.core.utils import native_value +class TimeStamp(TimeStamp): + """This schema is deprecated, you shoud use `kinto.core.schema.TimeStamp instead.""" + + def __init__(self, *args, **kwargs): + message = ("`kinto.core.resource.schema.TimeStamp` is deprecated, ", + "use `kinto.core.schema.TimeStamp` instead.") + warnings.warn(message, DeprecationWarning) + super(TimeStamp, self).__init__(*args, **kwargs) + + +class URL(URL): + """This schema is deprecated, you shoud use `kinto.core.schema.URL instead.""" + + def __init__(self, *args, **kwargs): + message = ("`kinto.core.resource.schema.URL` is deprecated, ", + "use `kinto.core.schema.URL` instead.") + warnings.warn(message, DeprecationWarning) + super(URL, self).__init__(*args, **kwargs) + + # Resource related schemas diff --git a/kinto/core/schema.py b/kinto/core/schema.py index 9e8d4e215..d8ddda796 100644 --- a/kinto/core/schema.py +++ b/kinto/core/schema.py @@ -1,3 +1,13 @@ +""" +This module contains generic schemas used by the core. You should declare schemas that +may be reused across the `kinto.core` here. + +.. note:: + + - If a schema is resource specific, you should use `kinto.core.resource.schema`. + - If a schema is view specific, you should declare it on the respective view. +""" + import six import colander diff --git a/tests/core/resource/test_schema.py b/tests/core/resource/test_schema.py index 0ace7da60..dc8857d7b 100644 --- a/tests/core/resource/test_schema.py +++ b/tests/core/resource/test_schema.py @@ -1,10 +1,28 @@ import six import colander +import mock from kinto.core.testing import unittest from kinto.core.resource import schema +class DepracatedSchemasTest(unittest.TestCase): + + def test_resource_timestamp_is_depracated(self): + with mock.patch('kinto.core.resource.schema.warnings') as mocked: + schema.TimeStamp() + message = ("`kinto.core.resource.schema.TimeStamp` is deprecated, ", + "use `kinto.core.schema.TimeStamp` instead.") + mocked.warn.assert_called_with(message, DeprecationWarning) + + def test_resource_URL_is_depracated(self): + with mock.patch('kinto.core.resource.schema.warnings') as mocked: + schema.URL() + message = ("`kinto.core.resource.schema.URL` is deprecated, ", + "use `kinto.core.schema.URL` instead.") + mocked.warn.assert_called_with(message, DeprecationWarning) + + class ResourceSchemaTest(unittest.TestCase): def test_preserves_unknown_fields_when_specified(self): From e6c3999966d799f25509d998712632256afe83bb Mon Sep 17 00:00:00 2001 From: Gabriela Surita Date: Tue, 7 Feb 2017 11:54:05 -0200 Subject: [PATCH 4/4] Fix docstrings --- kinto/core/resource/schema.py | 4 ++-- kinto/core/schema.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/kinto/core/resource/schema.py b/kinto/core/resource/schema.py index 666d60c60..195ed760e 100644 --- a/kinto/core/resource/schema.py +++ b/kinto/core/resource/schema.py @@ -8,7 +8,7 @@ class TimeStamp(TimeStamp): - """This schema is deprecated, you shoud use `kinto.core.schema.TimeStamp instead.""" + """This schema is deprecated, you shoud use `kinto.core.schema.TimeStamp` instead.""" def __init__(self, *args, **kwargs): message = ("`kinto.core.resource.schema.TimeStamp` is deprecated, ", @@ -18,7 +18,7 @@ def __init__(self, *args, **kwargs): class URL(URL): - """This schema is deprecated, you shoud use `kinto.core.schema.URL instead.""" + """This schema is deprecated, you shoud use `kinto.core.schema.URL` instead.""" def __init__(self, *args, **kwargs): message = ("`kinto.core.resource.schema.URL` is deprecated, ", diff --git a/kinto/core/schema.py b/kinto/core/schema.py index d8ddda796..f3ad39dac 100644 --- a/kinto/core/schema.py +++ b/kinto/core/schema.py @@ -1,8 +1,7 @@ -""" -This module contains generic schemas used by the core. You should declare schemas that +"""This module contains generic schemas used by the core. You should declare schemas that may be reused across the `kinto.core` here. -.. note:: +.. note:: This module is reserve for generic schemas only. - If a schema is resource specific, you should use `kinto.core.resource.schema`. - If a schema is view specific, you should declare it on the respective view.