From d6723879a394ed2836bd7470a4083ea1a833b628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 11 Apr 2023 18:38:00 -0600 Subject: [PATCH 1/5] feat: Allow `allowed_values` and `examples` in any JSON schema type constructor --- singer_sdk/typing.py | 123 ++++++++++++++++++++------ tests/core/test_jsonschema_helpers.py | 44 +++++++++ 2 files changed, 141 insertions(+), 26 deletions(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 3fe572ffa..56315c287 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -49,6 +49,7 @@ from typing import ( TYPE_CHECKING, Any, + Callable, Generator, Generic, ItemsView, @@ -62,7 +63,6 @@ import sqlalchemy from jsonschema import ValidationError, Validator, validators -from singer_sdk.helpers._classproperty import classproperty from singer_sdk.helpers._typing import ( JSONSCHEMA_ANNOTATION_SECRET, JSONSCHEMA_ANNOTATION_WRITEONLY, @@ -120,6 +120,9 @@ None, ] +T = TypeVar("T", bound=_JsonValue) +P = TypeVar("P") + def extend_validator_with_defaults(validator_class): # noqa: ANN001, ANN201 """Fill in defaults, before validating with the provided JSON Schema Validator. @@ -159,11 +162,57 @@ def set_defaults( ) -class JSONTypeHelper: +class DefaultInstanceProperty: + """Property of default instance. + + Descriptor similar to ``property`` that decorates an instance method to retrieve + a property from the instance initialized with default parameters, if the called on + the class. + """ + + def __init__(self, fget: Callable) -> None: + """Initialize the decorator. + + Args: + fget: The function to decorate. + """ + self.fget = fget + + def __get__(self, instance: P, owner: type[P]) -> Any: # noqa: ANN401 + """Get the property value. + + Args: + instance: The instance to get the property value from. + owner: The class to get the property value from. + + Returns: + The property value. + """ + if instance is None: + instance = owner() + return self.fget(instance) + + +class JSONTypeHelper(Generic[T]): """Type helper base class for JSONSchema types.""" - @classproperty - def type_dict(cls) -> dict: # noqa: N805 + def __init__( + self, + *, + allowed_values: list[T] | None = None, + examples: list[T] | None = None, + ) -> None: + """Initialize the type helper. + + Args: + allowed_values: A list of allowed values. + examples: A list of example values. + """ + self.allowed_values = allowed_values + self.examples = examples + + @DefaultInstanceProperty + def type_dict(self) -> dict: """Return dict describing the type. Raises: @@ -171,13 +220,29 @@ def type_dict(cls) -> dict: # noqa: N805 """ raise NotImplementedError + @property + def extras(self) -> dict: + """Return dict describing the JSON Schema extras. + + Returns: + A dictionary containing the JSON Schema extras. + """ + result = {} + if self.allowed_values: + result["enum"] = self.allowed_values + + if self.examples: + result["examples"] = self.examples + + return result + def to_dict(self) -> dict: """Convert to dictionary. Returns: A JSON Schema dictionary describing the object. """ - return cast(dict, self.type_dict) + return self.type_dict # type: ignore[no-any-return] def to_json(self, **kwargs: Any) -> str: """Convert to JSON. @@ -191,7 +256,7 @@ def to_json(self, **kwargs: Any) -> str: return json.dumps(self.to_dict(), **kwargs) -class StringType(JSONTypeHelper): +class StringType(JSONTypeHelper[str]): """String type.""" string_format: str | None = None @@ -206,12 +271,12 @@ class StringType(JSONTypeHelper): https://json-schema.org/understanding-json-schema/reference/string.html#built-in-formats """ - @classproperty - def _format(cls) -> dict: # noqa: N805 - return {"format": cls.string_format} if cls.string_format else {} + @property + def _format(self) -> dict: + return {"format": self.string_format} if self.string_format else {} - @classproperty - def type_dict(cls) -> dict: # noqa: N805 + @DefaultInstanceProperty + def type_dict(self) -> dict: """Get type dictionary. Returns: @@ -219,7 +284,8 @@ def type_dict(cls) -> dict: # noqa: N805 """ return { "type": ["string"], - **cls._format, + **self._format, + **self.extras, } @@ -328,58 +394,60 @@ class RegexType(StringType): string_format = "regex" -class BooleanType(JSONTypeHelper): +class BooleanType(JSONTypeHelper[bool]): """Boolean type.""" - @classproperty - def type_dict(cls) -> dict: # noqa: N805 + @DefaultInstanceProperty + def type_dict(self) -> dict: """Get type dictionary. Returns: A dictionary describing the type. """ - return {"type": ["boolean"]} + return {"type": ["boolean"], **self.extras} class IntegerType(JSONTypeHelper): """Integer type.""" - @classproperty - def type_dict(cls) -> dict: # noqa: N805 + @DefaultInstanceProperty + def type_dict(self) -> dict: """Get type dictionary. Returns: A dictionary describing the type. """ - return {"type": ["integer"]} + return {"type": ["integer"], **self.extras} -class NumberType(JSONTypeHelper): +class NumberType(JSONTypeHelper[float]): """Number type.""" - @classproperty - def type_dict(cls) -> dict: # noqa: N805 + @DefaultInstanceProperty + def type_dict(self) -> dict: """Get type dictionary. Returns: A dictionary describing the type. """ - return {"type": ["number"]} + return {"type": ["number"], **self.extras} W = TypeVar("W", bound=JSONTypeHelper) -class ArrayType(JSONTypeHelper, Generic[W]): +class ArrayType(JSONTypeHelper[list], Generic[W]): """Array type.""" - def __init__(self, wrapped_type: W | type[W]) -> None: + def __init__(self, wrapped_type: W | type[W], **kwargs: Any) -> None: """Initialize Array type with wrapped inner type. Args: wrapped_type: JSON Schema item type inside the array. + **kwargs: Additional keyword arguments to pass to the parent class. """ self.wrapped_type = wrapped_type + super().__init__(**kwargs) @property def type_dict(self) -> dict: # type: ignore[override] @@ -388,7 +456,7 @@ def type_dict(self) -> dict: # type: ignore[override] Returns: A dictionary describing the type. """ - return {"type": "array", "items": self.wrapped_type.type_dict} + return {"type": "array", "items": self.wrapped_type.type_dict, **self.extras} class Property(JSONTypeHelper, Generic[W]): @@ -491,6 +559,7 @@ def __init__( *properties: Property, additional_properties: W | type[W] | bool | None = None, pattern_properties: Mapping[str, W | type[W]] | None = None, + **kwargs: Any, ) -> None: """Initialize ObjectType from its list of properties. @@ -500,6 +569,7 @@ def __init__( this object, or a boolean indicating if extra properties are allowed. pattern_properties: A dictionary of regex patterns to match against property names, and the schema to match against the values. + **kwargs: Additional keyword arguments to pass to the `JSONTypeHelper`. Examples: >>> t = ObjectType( @@ -576,6 +646,7 @@ def __init__( self.wrapped: dict[str, Property] = {prop.name: prop for prop in properties} self.additional_properties = additional_properties self.pattern_properties = pattern_properties + super().__init__(**kwargs) @property def type_dict(self) -> dict: # type: ignore[override] diff --git a/tests/core/test_jsonschema_helpers.py b/tests/core/test_jsonschema_helpers.py index 6c262c412..34944205d 100644 --- a/tests/core/test_jsonschema_helpers.py +++ b/tests/core/test_jsonschema_helpers.py @@ -434,6 +434,50 @@ def test_inbuilt_type(json_type: JSONTypeHelper, expected_json_schema: dict): }, {is_integer_type}, ), + ( + Property( + "my_prop10", + ArrayType( + StringType( + allowed_values=["create", "delete", "insert", "update"], + examples=["insert", "update"], + ), + ), + ), + { + "my_prop10": { + "type": ["array", "null"], + "items": { + "type": ["string"], + "enum": ["create", "delete", "insert", "update"], + "examples": ["insert", "update"], + }, + }, + }, + {is_array_type, is_string_array_type}, + ), + ( + Property( + "my_prop11", + ArrayType( + StringType, + allowed_values=[ + ["create", "delete"], + ["insert", "update"], + ], + ), + ), + { + "my_prop11": { + "type": ["array", "null"], + "items": { + "type": ["string"], + }, + "enum": [["create", "delete"], ["insert", "update"]], + }, + }, + {is_array_type, is_string_array_type}, + ), ], ) def test_property_creation( From 2b7951c08743f8d5d39c8862092497c96bf44856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 12 Apr 2023 09:49:46 -0600 Subject: [PATCH 2/5] Update `Property` generics --- singer_sdk/typing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 56315c287..aefda68fe 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -459,20 +459,20 @@ def type_dict(self) -> dict: # type: ignore[override] return {"type": "array", "items": self.wrapped_type.type_dict, **self.extras} -class Property(JSONTypeHelper, Generic[W]): +class Property(JSONTypeHelper[T], Generic[T]): """Generic Property. Should be nested within a `PropertiesList`.""" # TODO: Make some of these arguments keyword-only. This is a breaking change. def __init__( self, name: str, - wrapped: W | type[W], + wrapped: JSONTypeHelper[T] | type[JSONTypeHelper[T]], required: bool = False, # noqa: FBT001, FBT002 - default: _JsonValue | None = None, + default: T | None = None, description: str | None = None, secret: bool | None = False, # noqa: FBT002 - allowed_values: list[Any] | None = None, - examples: list[Any] | None = None, + allowed_values: list[T] | None = None, + examples: list[T] | None = None, ) -> None: """Initialize Property object. @@ -661,7 +661,7 @@ def type_dict(self) -> dict: # type: ignore[override] merged_props.update(w.to_dict()) if not w.optional: required.append(w.name) - result: dict = {"type": "object", "properties": merged_props} + result: dict[str, Any] = {"type": "object", "properties": merged_props} if required: result["required"] = required From 97cc709333cf434fda042259fdb90b789763deb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 12 Apr 2023 09:53:50 -0600 Subject: [PATCH 3/5] Add usage exmaples --- singer_sdk/typing.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index aefda68fe..1bc11ba7f 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -10,6 +10,15 @@ Property("id", IntegerType, required=True), Property("foo_or_bar", StringType, allowed_values=["foo", "bar"]), + Property( + "permissions", + ArrayType( + th.StringType( + allowed_values=["create", "delete", "insert", "update"], + examples=["insert", "update"], + ), + ), + ), Property("ratio", NumberType, examples=[0.25, 0.75, 1.0]), Property("days_active", IntegerType), Property("updated_on", DateTimeType), @@ -257,7 +266,16 @@ def to_json(self, **kwargs: Any) -> str: class StringType(JSONTypeHelper[str]): - """String type.""" + """String type. + + Examples: + >>> StringType.type_dict + {'type': ['string']} + >>> StringType().type_dict + {'type': ['string']} + >>> StringType(allowed_values=["a", "b"]).type_dict + {'type': ['string'], 'enum': ['a', 'b']} + """ string_format: str | None = None """String format. From 905a77ebe36dccfe8689d66a94202ad7f9726a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 12 Apr 2023 10:41:24 -0600 Subject: [PATCH 4/5] Remove `th.` from call path --- singer_sdk/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 1bc11ba7f..6e4226169 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -13,7 +13,7 @@ Property( "permissions", ArrayType( - th.StringType( + StringType( allowed_values=["create", "delete", "insert", "update"], examples=["insert", "update"], ), From f792ee21f713a40943ac2f9ab147091a0bc3e964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Wed, 12 Apr 2023 13:52:52 -0600 Subject: [PATCH 5/5] Add doctests for other type helper classes --- singer_sdk/typing.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/singer_sdk/typing.py b/singer_sdk/typing.py index 6e4226169..f73c8bd4f 100644 --- a/singer_sdk/typing.py +++ b/singer_sdk/typing.py @@ -413,7 +413,14 @@ class RegexType(StringType): class BooleanType(JSONTypeHelper[bool]): - """Boolean type.""" + """Boolean type. + + Examples: + >>> BooleanType.type_dict + {'type': ['boolean']} + >>> BooleanType().type_dict + {'type': ['boolean']} + """ @DefaultInstanceProperty def type_dict(self) -> dict: @@ -426,7 +433,16 @@ def type_dict(self) -> dict: class IntegerType(JSONTypeHelper): - """Integer type.""" + """Integer type. + + Examples: + >>> IntegerType.type_dict + {'type': ['integer']} + >>> IntegerType().type_dict + {'type': ['integer']} + >>> IntegerType(allowed_values=[1, 2]).type_dict + {'type': ['integer'], 'enum': [1, 2]} + """ @DefaultInstanceProperty def type_dict(self) -> dict: @@ -439,7 +455,16 @@ def type_dict(self) -> dict: class NumberType(JSONTypeHelper[float]): - """Number type.""" + """Number type. + + Examples: + >>> NumberType.type_dict + {'type': ['number']} + >>> NumberType().type_dict + {'type': ['number']} + >>> NumberType(allowed_values=[1.0, 2.0]).type_dict + {'type': ['number'], 'enum': [1.0, 2.0]} + """ @DefaultInstanceProperty def type_dict(self) -> dict: