From 531b9ccfd9cb5f6105abbca961086a4215d5175d Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 29 Nov 2022 13:40:11 -0500 Subject: [PATCH 1/5] support integer-1.1 and ndarray-1.1 fixes an issue with integer where InterType instead of cls was used during class instantiation which resulted in objects being created with the incorrect version. Prior code (that only had one integer version, 1.0) did not suffer from issues because of this bug. fixes #1245 by removing the use of super in NDArrayType and adding an exception to check for incompatible use of super in classes of type ExtensionTypeMeta. --- asdf/tags/core/integer.py | 3 ++- asdf/tags/core/ndarray.py | 11 ++++++++--- asdf/types.py | 9 +++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index 574ff12df..ec8eeaa4c 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -44,6 +44,7 @@ class IntegerType(AsdfType): name = "core/integer" version = "1.0.0" + supported_versions = {"1.0.0", "1.1.0"} _value_cache = dict() @@ -96,7 +97,7 @@ def from_tree(cls, tree, ctx): if tree["sign"] == "-": value = -value - return IntegerType(value) + return cls(value) def __int__(self): return int(self._value) diff --git a/asdf/tags/core/ndarray.py b/asdf/tags/core/ndarray.py index 48e366106..5fb217a2e 100644 --- a/asdf/tags/core/ndarray.py +++ b/asdf/tags/core/ndarray.py @@ -214,6 +214,7 @@ def ascii_to_unicode(x): class NDArrayType(AsdfType): name = "core/ndarray" version = "1.0.0" + supported_versions = {"1.0.0", "1.1.0"} types = [np.ndarray, ma.MaskedArray] def __init__(self, source, shape, dtype, offset, strides, order, mask, asdffile): @@ -372,10 +373,10 @@ def __getattribute__(self, name): # can cause problems when the array is passed to other # libraries. # See https://github.com/asdf-format/asdf/issues/1015 - if name in ("name", "version"): + if name in ("name", "version", "supported_versions"): raise AttributeError(f"'{self.__class__.name}' object has no attribute '{name}'") else: - return super().__getattribute__(name) + return AsdfType.__getattribute__(self, name) @classmethod def from_tree(cls, node, ctx): @@ -564,6 +565,9 @@ def __operation__(self, *args): return __operation__ +classes_to_modify = NDArrayType.__versioned_siblings + [ + NDArrayType, +] for op in [ "__neg__", "__pos__", @@ -628,7 +632,8 @@ def __operation__(self, *args): "__delitem__", "__contains__", ]: - setattr(NDArrayType, op, _make_operation(op)) + [setattr(cls, op, _make_operation(op)) for cls in classes_to_modify] +del classes_to_modify def _get_ndim(instance): diff --git a/asdf/types.py b/asdf/types.py index 129e41073..7a89d0abe 100644 --- a/asdf/types.py +++ b/asdf/types.py @@ -133,6 +133,15 @@ def __new__(mcls, name, bases, attrs): new_attrs["version"] = version new_attrs["supported_versions"] = set() new_attrs["_latest_version"] = cls.version + if "__classcell__" in new_attrs: + raise RuntimeError( + "Subclasses of ExtensionTypeMeta that define " + "supported_versions cannot used super() to call " + "parent class functions. super() creates a " + "__classcell__ closure that cannot be duplicated " + "during creation of versioned siblings. " + "See https://github.com/asdf-format/asdf/issues/1245" + ) siblings.append(ExtensionTypeMeta.__new__(mcls, name, bases, new_attrs)) setattr(cls, "__versioned_siblings", siblings) From cb98823c4996a5de820bf8ed277f4155dcffb34a Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 30 Nov 2022 12:37:38 -0500 Subject: [PATCH 2/5] remove xfailss for 1.6 tags in test_version_map_support --- asdf/tests/test_versioning.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/asdf/tests/test_versioning.py b/asdf/tests/test_versioning.py index d7afd2958..f7d1f8847 100644 --- a/asdf/tests/test_versioning.py +++ b/asdf/tests/test_versioning.py @@ -294,12 +294,9 @@ def xfail_version_map_support_cases(request): version = request.getfixturevalue("version") if (version, tag) in [ ("1.6.0", "tag:stsci.edu:asdf/core/column-1.1.0"), - ("1.6.0", "tag:stsci.edu:asdf/core/integer-1.1.0"), - ("1.6.0", "tag:stsci.edu:asdf/core/ndarray-1.1.0"), ("1.6.0", "tag:stsci.edu:asdf/core/table-1.1.0"), ("1.6.0", "tag:stsci.edu:asdf/fits/fits-1.1.0"), ("1.6.0", "tag:stsci.edu:asdf/time/time-1.2.0"), - ("1.6.0", "tag:stsci.edu:asdf/unit/defunit-1.0.0"), ("1.6.0", "tag:stsci.edu:asdf/unit/defunit-1.1.0"), ("1.6.0", "tag:stsci.edu:asdf/unit/quantity-1.2.0"), ("1.6.0", "tag:stsci.edu:asdf/unit/unit-1.1.0"), From c50f265ac35f24aaccdbe67ba2830875026aa592 Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 30 Nov 2022 12:39:44 -0500 Subject: [PATCH 3/5] update changelog --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index 76291d83b..363f729ce 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -19,6 +19,7 @@ The ASDF Standard is at v1.6.0 The ASDF Standard is at v1.6.0 - Fix issue #1239, close memmap with asdf file context [#1241] +- Add ndarray-1.1.0 and integer-1.1.0 support [#1250] 2.14.0 (2022-11-22) ------------------- From 7ea4f9f2d7cc2921347fc94ef0f5d9ace4a5862e Mon Sep 17 00:00:00 2001 From: Brett Date: Wed, 30 Nov 2022 16:02:20 -0500 Subject: [PATCH 4/5] skip defunit-1.0 in test_version_map for asdf-standard 1.6 --- asdf/tests/test_versioning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/asdf/tests/test_versioning.py b/asdf/tests/test_versioning.py index f7d1f8847..d189f8270 100644 --- a/asdf/tests/test_versioning.py +++ b/asdf/tests/test_versioning.py @@ -298,6 +298,7 @@ def xfail_version_map_support_cases(request): ("1.6.0", "tag:stsci.edu:asdf/fits/fits-1.1.0"), ("1.6.0", "tag:stsci.edu:asdf/time/time-1.2.0"), ("1.6.0", "tag:stsci.edu:asdf/unit/defunit-1.1.0"), + ("1.6.0", "tag:stsci.edu:asdf/unit/defunit-1.0.0"), ("1.6.0", "tag:stsci.edu:asdf/unit/quantity-1.2.0"), ("1.6.0", "tag:stsci.edu:asdf/unit/unit-1.1.0"), ("1.5.0", "tag:stsci.edu:asdf/unit/defunit-1.0.0"), From 80d5904bbad42e90c07461e327456b2b41d25c69 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 13 Dec 2022 15:49:51 -0500 Subject: [PATCH 5/5] add test for issue #1245 --- asdf/tests/test_types.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/asdf/tests/test_types.py b/asdf/tests/test_types.py index 5ba4472e8..84be623dc 100644 --- a/asdf/tests/test_types.py +++ b/asdf/tests/test_types.py @@ -655,3 +655,36 @@ def test_custom_reference_cycle(tmp_path): with asdf.open(path, extensions=FractionWithInverseExtension()) as af: assert af["fraction"].inverse.inverse is af["fraction"] + + +def test_super_use_in_versioned_subclass(): + """ + Test fix for issue: https://github.com/asdf-format/asdf/issues/1245 + + Legacy extensions cannot use super in subclasses of CustomType + that define supported_versions due to the metaclasses inability + to create distinct __classcell__ closures. + """ + + class Foo: + def __init__(self, bar): + self.bar = bar + + with pytest.raises(RuntimeError, match=r".* ExtensionTypeMeta .* __classcell__ .*"): + + class FooType(asdf.CustomType): + name = "foo" + version = (1, 0, 0) + supported_versions = [(1, 1, 0), (1, 2, 0)] + types = [Foo] + + @classmethod + def to_tree(cls, node, ctx): + return {"bar": node.bar} + + @classmethod + def from_tree(cls, tree, ctx): + return Foo(tree["bar"]) + + def __getattribute__(self, name): + return super().__getattribute__(name)