From 9f01ab9ba4a0f0882d6cb5f917b38a8ae56fa12f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Tue, 3 Jan 2023 14:51:40 +0100 Subject: [PATCH 1/5] Override hooks --- src/cattrs/gen.py | 88 +++++++++++++++++++++++++++--------------- tests/test_gen_dict.py | 37 ++++++++++++++++++ 2 files changed, 93 insertions(+), 32 deletions(-) diff --git a/src/cattrs/gen.py b/src/cattrs/gen.py index 23601f47..bde927e0 100644 --- a/src/cattrs/gen.py +++ b/src/cattrs/gen.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import linecache import re import uuid @@ -17,7 +19,7 @@ ) import attr -from attr import NOTHING, frozen, resolve_types +from attr import NOTHING, Attribute, frozen, resolve_types from cattrs.errors import ( ClassValidationError, @@ -45,14 +47,18 @@ class AttributeOverride: omit_if_default: Optional[bool] = None rename: Optional[str] = None omit: bool = False # Omit the field completely. + struct_hook: Optional[Callable[[Any, Any], Any]] = None # Structure hook to use. + unstruct_hook: Optional[Callable[[Any], Any]] = None # Structure hook to use. def override( omit_if_default: Optional[bool] = None, rename: Optional[str] = None, omit: bool = False, + struct_hook: Optional[Callable[[Any, Any], Any]] = None, + unstruct_hook: Optional[Callable[[Any], Any]] = None, ): - return AttributeOverride(omit_if_default=omit_if_default, rename=rename, omit=omit) + return AttributeOverride(omit_if_default, rename, omit, struct_hook, unstruct_hook) _neutral = AttributeOverride() @@ -120,24 +126,27 @@ def make_dict_unstructure_fn( # If a type is manually overwritten, this function should be # regenerated. handler = None - if a.type is not None: - t = a.type - if isinstance(t, TypeVar): - if t.__name__ in mapping: - t = mapping[t.__name__] - else: - handler = converter.unstructure - elif is_generic(t) and not is_bare(t) and not is_annotated(t): - t = deep_copy_with(t, mapping) - - if handler is None: - try: - handler = converter._unstructure_func.dispatch(t) - except RecursionError: - # There's a circular reference somewhere down the line - handler = converter.unstructure + if override.unstruct_hook is not None: + handler = override.unstruct_hook else: - handler = converter.unstructure + if a.type is not None: + t = a.type + if isinstance(t, TypeVar): + if t.__name__ in mapping: + t = mapping[t.__name__] + else: + handler = converter.unstructure + elif is_generic(t) and not is_bare(t) and not is_annotated(t): + t = deep_copy_with(t, mapping) + + if handler is None: + try: + handler = converter._unstructure_func.dispatch(t) + except RecursionError: + # There's a circular reference somewhere down the line + handler = converter.unstructure + else: + handler = converter.unstructure is_identity = handler == converter._unstructure_identity @@ -229,6 +238,28 @@ def _generate_mapping(cl: Type, old_mapping: Dict[str, type]) -> Dict[str, type] return mapping +def find_structure_handler( + a: Attribute, type: Any, c: BaseConverter, prefer_attrs_converters: bool = False +) -> Optional[Callable[[Any, Any], Any]]: + """Find the appropriate structure handler to use. + + Return `None` if no handler should be used. + """ + if a.converter is not None and prefer_attrs_converters: + # If the user as requested to use attrib converters, use nothing + # so it falls back to that. + handler = None + elif a.converter is not None and not prefer_attrs_converters and type is not None: + handler = c._structure_func.dispatch(type) + if handler == c._structure_error: + handler = None + elif type is not None: + handler = c._structure_func.dispatch(type) + else: + handler = c.structure + return handler + + DictStructureFn = Callable[[Mapping[str, Any], Any], T] @@ -311,20 +342,13 @@ def make_dict_structure_fn( # For each attribute, we try resolving the type here and now. # If a type is manually overwritten, this function should be # regenerated. - if a.converter is not None and _cattrs_prefer_attrib_converters: - handler = None - elif ( - a.converter is not None - and not _cattrs_prefer_attrib_converters - and t is not None - ): - handler = converter._structure_func.dispatch(t) - if handler == converter._structure_error: - handler = None - elif t is not None: - handler = converter._structure_func.dispatch(t) + if override.struct_hook is not None: + # If the user has requested an override, just use that. + handler = override.struct_hook else: - handler = converter.structure + handler = find_structure_handler( + a, t, converter, _cattrs_prefer_attrib_converters + ) struct_handler_name = f"__c_structure_{an}" internal_arg_parts[struct_handler_name] = handler diff --git a/tests/test_gen_dict.py b/tests/test_gen_dict.py index 6f9d5cff..74ccbd2c 100644 --- a/tests/test_gen_dict.py +++ b/tests/test_gen_dict.py @@ -304,3 +304,40 @@ def test_type_names_with_quotes(): assert converter.structure( {2: "a"}, Dict[Union[Literal["a", 2, 3], Literal[4]], str] ) == {2: "a"} + + +def test_overriding_struct_hook(converter: BaseConverter) -> None: + """Overriding structure hooks works.""" + from math import ceil + + @define + class A: + a: int + b: str + + converter.register_structure_hook( + A, + make_dict_structure_fn( + A, converter, a=override(struct_hook=lambda v, _: ceil(v)) + ), + ) + + assert converter.structure({"a": 0.5, "b": 1}, A) == A(1, "1") + + +def test_overriding_unstruct_hook(converter: BaseConverter) -> None: + """Overriding unstructure hooks works.""" + + @define + class A: + a: int + b: str + + converter.register_unstructure_hook( + A, + make_dict_unstructure_fn( + A, converter, a=override(unstruct_hook=lambda v: v + 1) + ), + ) + + assert converter.unstructure(A(1, "")) == {"a": 2, "b": ""} From 888535d299c4f29e411ebea59b7c736b3d799463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Tue, 3 Jan 2023 14:52:43 +0100 Subject: [PATCH 2/5] Update css --- docs/_static/custom.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 86aa2a9a..e0bbeb2f 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -133,3 +133,7 @@ div.tab-set { div.tab-set pre { padding: 1.25em; } + +body:not([data-theme="light"]) .highlight { + background: #18181a; +} From 8b88a46b0ef43a4ca6418d89a695d24b64cb0573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Tue, 3 Jan 2023 15:38:49 +0100 Subject: [PATCH 3/5] Docs --- HISTORY.rst | 1 + docs/customizing.md | 185 +++++++++++++++++++++++++++++++++++++++++++ docs/customizing.rst | 166 -------------------------------------- 3 files changed, 186 insertions(+), 166 deletions(-) create mode 100644 docs/customizing.md delete mode 100644 docs/customizing.rst diff --git a/HISTORY.rst b/HISTORY.rst index 046f75e0..ab35198a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,7 @@ History 22.3.0 (UNRELEASED) ------------------- * Introduce the `tagged_union` strategy. (`#318 `_ `#317 `_) +* Introduce `override.struct_hook` and `override.unstruct_hook`. Learn more `here `_. 22.2.0 (2022-10-03) ------------------- diff --git a/docs/customizing.md b/docs/customizing.md new file mode 100644 index 00000000..b42dbd92 --- /dev/null +++ b/docs/customizing.md @@ -0,0 +1,185 @@ +# Customizing class un/structuring + +This section deals with customizing the unstructuring and structuring processes in `cattrs`. + +## Using `cattr.Converter` + +The default `Converter`, upon first encountering an `attrs` class, will use +the generation functions mentioned here to generate the specialized hooks for it, +register the hooks and use them. + +## Manual un/structuring hooks + +You can write your own structuring and unstructuring functions and register +them for types using {meth}`Converter.register_structure_hook ` and +{meth}`Converter.register_unstructure_hook `. This approach is the most +flexible but also requires the most amount of boilerplate. + +## Using `cattrs.gen` generators + +`cattrs` includes a module, {mod}`cattrs.gen`, which allows for generating and +compiling specialized functions for unstructuring `attrs` classes. + +One reason for generating these functions in advance is that they can bypass +a lot of `cattrs` machinery and be significantly faster than normal `cattrs`. + +Another reason is that it's possible to override behavior on a per-attribute basis. + +Currently, the overrides only support generating dictionary un/structuring functions +(as opposed to tuples), and support `omit_if_default`, `forbid_extra_keys`, +`rename` and `omit`. + +### `omit_if_default` + +This override can be applied on a per-class or per-attribute basis. The generated +unstructuring function will skip unstructuring values that are equal to their +default or factory values. + +```{doctest} + +>>> from cattrs.gen import make_dict_unstructure_fn, override +>>> +>>> @define +... class WithDefault: +... a: int +... b: dict = Factory(dict) +>>> +>>> c = cattrs.Converter() +>>> c.register_unstructure_hook(WithDefault, make_dict_unstructure_fn(WithDefault, c, b=override(omit_if_default=True))) +>>> c.unstructure(WithDefault(1)) +{'a': 1} +``` + +Note that the per-attribute value overrides the per-class value. A side-effect +of this is the ability to force the presence of a subset of fields. +For example, consider a class with a `DateTime` field and a factory for it: +skipping the unstructuring of the `DateTime` field would be inconsistent and +based on the current time. So we apply the `omit_if_default` rule to the class, +but not to the `DateTime` field. + +```{note} + The parameter to `make_dict_unstructure_function` is named ``_cattrs_omit_if_default`` instead of just ``omit_if_default`` to avoid potential collisions with an override for a field named ``omit_if_default``. +``` + +```{doctest} + +>>> from pendulum import DateTime +>>> from cattrs.gen import make_dict_unstructure_fn, override +>>> +>>> @define +... class TestClass: +... a: Optional[int] = None +... b: DateTime = Factory(DateTime.utcnow) +>>> +>>> c = cattrs.Converter() +>>> hook = make_dict_unstructure_fn(TestClass, c, _cattrs_omit_if_default=True, b=override(omit_if_default=False)) +>>> c.register_unstructure_hook(TestClass, hook) +>>> c.unstructure(TestClass()) +{'b': ...} +``` + +This override has no effect when generating structuring functions. + +### `forbid_extra_keys` + +By default `cattrs` is lenient in accepting unstructured input. If extra +keys are present in a dictionary, they will be ignored when generating a +structured object. Sometimes it may be desirable to enforce a stricter +contract, and to raise an error when unknown keys are present - in particular +when fields have default values this may help with catching typos. +`forbid_extra_keys` can also be enabled (or disabled) on a per-class basis when +creating structure hooks with `make_dict_structure_fn`. + +```{doctest} + :options: +SKIP + +>>> from cattrs.gen import make_dict_structure_fn +>>> +>>> @define +... class TestClass: +... number: int = 1 +>>> +>>> c = cattrs.Converter(forbid_extra_keys=True) +>>> c.structure({"nummber": 2}, TestClass) +Traceback (most recent call last): +... +ForbiddenExtraKeyError: Extra fields in constructor for TestClass: nummber +>>> hook = make_dict_structure_fn(TestClass, c, _cattrs_forbid_extra_keys=False) +>>> c.register_structure_hook(TestClass, hook) +>>> c.structure({"nummber": 2}, TestClass) +TestClass(number=1) +``` + +This behavior can only be applied to classes or to the default for the +`Converter`, and has no effect when generating unstructuring functions. + +### `rename` + +Using the rename override makes `cattrs` simply use the provided name instead +of the real attribute name. This is useful if an attribute name is a reserved +keyword in Python. + +```{doctest} + +>>> from pendulum import DateTime +>>> from cattrs.gen import make_dict_unstructure_fn, make_dict_structure_fn, override +>>> +>>> @define +... class ExampleClass: +... klass: Optional[int] +>>> +>>> c = cattrs.Converter() +>>> unst_hook = make_dict_unstructure_fn(ExampleClass, c, klass=override(rename="class")) +>>> st_hook = make_dict_structure_fn(ExampleClass, c, klass=override(rename="class")) +>>> c.register_unstructure_hook(ExampleClass, unst_hook) +>>> c.register_structure_hook(ExampleClass, st_hook) +>>> c.unstructure(ExampleClass(1)) +{'class': 1} +>>> c.structure({'class': 1}, ExampleClass) +ExampleClass(klass=1) +``` + +### `omit` + +This override can only be applied to individual attributes. Using the `omit` +override will simply skip the attribute completely when generating a structuring +or unstructuring function. + +```{doctest} + +>>> from cattrs.gen import make_dict_unstructure_fn, override +>>> +>>> @define +... class ExampleClass: +... an_int: int +>>> +>>> c = cattrs.Converter() +>>> unst_hook = make_dict_unstructure_fn(ExampleClass, c, an_int=override(omit=True)) +>>> c.register_unstructure_hook(ExampleClass, unst_hook) +>>> c.unstructure(ExampleClass(1)) +{} +``` + +### `struct_hook` and `unstruct_hook` + +By default, the generators will determine the right un/structure hook for each attribute of a class at time of generation according to the type of each individual attribute. + +This process can be overriden by passing in the desired un/structure manually. + +```{doctest} + +>>> from cattrs.gen import make_dict_structure_fn, override + +>>> @define +... class ExampleClass: +... an_int: int + +>>> c = cattrs.Converter() +>>> st_hook = make_dict_structure_fn( +... ExampleClass, c, an_int=override(struct_hook=lambda v, _: v + 1) +... ) +>>> c.register_structure_hook(ExampleClass, st_hook) + +>>> c.structure({"an_int": 1}, ExampleClass) +ExampleClass(an_int=2) +``` diff --git a/docs/customizing.rst b/docs/customizing.rst deleted file mode 100644 index f1761db4..00000000 --- a/docs/customizing.rst +++ /dev/null @@ -1,166 +0,0 @@ -================================ -Customizing class un/structuring -================================ - -This section deals with customizing the unstructuring and structuring processes in `cattrs`. - -Using ``cattr.Converter`` -************************* - -The default ``Converter``, upon first encountering an ``attrs`` class, will use -the generation functions mentioned here to generate the specialized hooks for it, -register the hooks and use them. - -Manual un/structuring hooks -*************************** - -You can write your own structuring and unstructuring functions and register -them for types using :py:attr:`Converter.register_structure_hook ` and -:py:attr:`Converter.register_unstructure_hook `. This approach is the most -flexible but also requires the most amount of boilerplate. - -Using ``cattrs.gen`` generators -******************************* - -`cattrs` includes a module, :mod:`cattrs.gen`, which allows for generating and -compiling specialized functions for unstructuring ``attrs`` classes. - -One reason for generating these functions in advance is that they can bypass -a lot of `cattrs` machinery and be significantly faster than normal `cattrs`. - -Another reason is that it's possible to override behavior on a per-attribute basis. - -Currently, the overrides only support generating dictionary un/structuring functions -(as opposed to tuples), and support ``omit_if_default``, ``forbid_extra_keys``, -``rename`` and ``omit``. - -``omit_if_default`` -------------------- - -This override can be applied on a per-class or per-attribute basis. The generated -unstructuring function will skip unstructuring values that are equal to their -default or factory values. - -.. doctest:: - - >>> from cattrs.gen import make_dict_unstructure_fn, override - >>> - >>> @define - ... class WithDefault: - ... a: int - ... b: dict = Factory(dict) - >>> - >>> c = cattrs.Converter() - >>> c.register_unstructure_hook(WithDefault, make_dict_unstructure_fn(WithDefault, c, b=override(omit_if_default=True))) - >>> c.unstructure(WithDefault(1)) - {'a': 1} - -Note that the per-attribute value overrides the per-class value. A side-effect -of this is the ability to force the presence of a subset of fields. -For example, consider a class with a `DateTime` field and a factory for it: -skipping the unstructuring of the `DateTime` field would be inconsistent and -based on the current time. So we apply the `omit_if_default` rule to the class, -but not to the `DateTime` field. - -.. note:: - - The parameter to `make_dict_unstructure_function` is named ``_cattrs_omit_if_default`` instead of just ``omit_if_default`` to avoid potential collisions with an override for a field named ``omit_if_default``. - -.. doctest:: - - >>> from pendulum import DateTime - >>> from cattrs.gen import make_dict_unstructure_fn, override - >>> - >>> @define - ... class TestClass: - ... a: Optional[int] = None - ... b: DateTime = Factory(DateTime.utcnow) - >>> - >>> c = cattrs.Converter() - >>> hook = make_dict_unstructure_fn(TestClass, c, _cattrs_omit_if_default=True, b=override(omit_if_default=False)) - >>> c.register_unstructure_hook(TestClass, hook) - >>> c.unstructure(TestClass()) - {'b': ...} - -This override has no effect when generating structuring functions. - -``forbid_extra_keys`` ---------------------- - -By default ``cattrs`` is lenient in accepting unstructured input. If extra -keys are present in a dictionary, they will be ignored when generating a -structured object. Sometimes it may be desirable to enforce a stricter -contract, and to raise an error when unknown keys are present - in particular -when fields have default values this may help with catching typos. -`forbid_extra_keys` can also be enabled (or disabled) on a per-class basis when -creating structure hooks with ``make_dict_structure_fn``. - -.. doctest:: - :options: +SKIP - - >>> from cattrs.gen import make_dict_structure_fn - >>> - >>> @define - ... class TestClass: - ... number: int = 1 - >>> - >>> c = cattrs.Converter(forbid_extra_keys=True) - >>> c.structure({"nummber": 2}, TestClass) - Traceback (most recent call last): - ... - ForbiddenExtraKeyError: Extra fields in constructor for TestClass: nummber - >>> hook = make_dict_structure_fn(TestClass, c, _cattrs_forbid_extra_keys=False) - >>> c.register_structure_hook(TestClass, hook) - >>> c.structure({"nummber": 2}, TestClass) - TestClass(number=1) - -This behavior can only be applied to classes or to the default for the -`Converter`, and has no effect when generating unstructuring functions. - -``rename`` ----------- - -Using the rename override makes ``cattrs`` simply use the provided name instead -of the real attribute name. This is useful if an attribute name is a reserved -keyword in Python. - -.. doctest:: - - >>> from pendulum import DateTime - >>> from cattrs.gen import make_dict_unstructure_fn, make_dict_structure_fn, override - >>> - >>> @define - ... class ExampleClass: - ... klass: Optional[int] - >>> - >>> c = cattrs.Converter() - >>> unst_hook = make_dict_unstructure_fn(ExampleClass, c, klass=override(rename="class")) - >>> st_hook = make_dict_structure_fn(ExampleClass, c, klass=override(rename="class")) - >>> c.register_unstructure_hook(ExampleClass, unst_hook) - >>> c.register_structure_hook(ExampleClass, st_hook) - >>> c.unstructure(ExampleClass(1)) - {'class': 1} - >>> c.structure({'class': 1}, ExampleClass) - ExampleClass(klass=1) - -``omit`` --------- - -This override can only be applied to individual attributes. Using the ``omit`` -override will simply skip the attribute completely when generating a structuring -or unstructuring function. - - -.. doctest:: - - >>> from cattrs.gen import make_dict_unstructure_fn, override - >>> - >>> @define - ... class ExampleClass: - ... an_int: int - >>> - >>> c = cattrs.Converter() - >>> unst_hook = make_dict_unstructure_fn(ExampleClass, c, an_int=override(omit=True)) - >>> c.register_unstructure_hook(ExampleClass, unst_hook) - >>> c.unstructure(ExampleClass(1)) - {} From 7bff9512547caeb9388be566930a882b0557ea52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Tue, 3 Jan 2023 16:01:18 +0100 Subject: [PATCH 4/5] Tweak GitHub --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 902de26c..e5d9e265 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,11 +13,11 @@ jobs: name: "Python ${{ matrix.python-version }}" runs-on: "ubuntu-latest" env: - USING_COVERAGE: "3.7,3.8,3.9,3.10,pypy-3.8" + USING_COVERAGE: "3.7,3.8,3.9,3.10,3.11,pypy-3.8" strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11.0-rc.2", "pypy-3.8"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.8"] steps: - uses: "actions/checkout@v2" From b6d6a657a0f8b72de0e693d43a0ea949a55d7e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Tue, 3 Jan 2023 16:02:20 +0100 Subject: [PATCH 5/5] Tweak HISTORY --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index ab35198a..af7ae120 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,7 @@ History ------------------- * Introduce the `tagged_union` strategy. (`#318 `_ `#317 `_) * Introduce `override.struct_hook` and `override.unstruct_hook`. Learn more `here `_. + (`#326 `_) 22.2.0 (2022-10-03) -------------------