diff --git a/CHANGES.rst b/CHANGES.rst index af3d73d6f..71e852218 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,7 @@ The ASDF Standard is at v1.6.0 - Add AsdfProvisionalAPIWarning to warn developers of new features that may undergo breaking changes but are likely to be included as stable features (without this warning) in a future version of ASDF [#1295] +- Add new plugin type for custom schema validators. [#1328] 2.14.3 (2022-12-15) ------------------- diff --git a/asdf/core/__init__.py b/asdf/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/asdf/core/_extensions.py b/asdf/core/_extensions.py new file mode 100644 index 000000000..e0b06e245 --- /dev/null +++ b/asdf/core/_extensions.py @@ -0,0 +1,23 @@ +from asdf.extension import ManifestExtension + +from ._validators import ndarray + +VALIDATORS = [ + ndarray.NdimValidator(), + ndarray.MaxNdimValidator(), + ndarray.DatatypeValidator(), +] + + +MANIFEST_URIS = [ + "asdf://asdf-format.org/core/manifests/core-1.0.0", + "asdf://asdf-format.org/core/manifests/core-1.1.0", + "asdf://asdf-format.org/core/manifests/core-1.2.0", + "asdf://asdf-format.org/core/manifests/core-1.3.0", + "asdf://asdf-format.org/core/manifests/core-1.4.0", + "asdf://asdf-format.org/core/manifests/core-1.5.0", + "asdf://asdf-format.org/core/manifests/core-1.6.0", +] + + +EXTENSIONS = [ManifestExtension.from_uri(u, validators=VALIDATORS) for u in MANIFEST_URIS] diff --git a/asdf/core/_integration.py b/asdf/core/_integration.py new file mode 100644 index 000000000..43ea1640a --- /dev/null +++ b/asdf/core/_integration.py @@ -0,0 +1,21 @@ +from asdf.resource import JsonschemaResourceMapping + + +def get_extensions(): + """ + Get the extension instances for the core extensions. This method is registered with the + asdf.extensions entry point. + + Returns + ------- + list of asdf.extension.Extension + """ + from . import _extensions + + return _extensions.EXTENSIONS + + +def get_json_schema_resource_mappings(): + return [ + JsonschemaResourceMapping(), + ] diff --git a/asdf/core/_validators/__init__.py b/asdf/core/_validators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/asdf/core/_validators/ndarray.py b/asdf/core/_validators/ndarray.py new file mode 100644 index 000000000..f9a089da1 --- /dev/null +++ b/asdf/core/_validators/ndarray.py @@ -0,0 +1,28 @@ +from asdf.extension import Validator +from asdf.tags.core.ndarray import validate_datatype, validate_max_ndim, validate_ndim + + +class NdimValidator(Validator): + schema_property = "ndim" + # The validators in this module should really only be applied + # to ndarray-* tags, but that will have to be a 3.0 change. + tags = ["**"] + + def validate(self, expected_ndim, node, schema): + yield from validate_ndim(None, expected_ndim, node, schema) + + +class MaxNdimValidator(Validator): + schema_property = "max_ndim" + tags = ["**"] + + def validate(self, max_ndim, node, schema): + yield from validate_max_ndim(None, max_ndim, node, schema) + + +class DatatypeValidator(Validator): + schema_property = "datatype" + tags = ["**"] + + def validate(self, expected_datatype, node, schema): + yield from validate_datatype(None, expected_datatype, node, schema) diff --git a/asdf/core/tests/__init__.py b/asdf/core/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/asdf/core/tests/test_integration.py b/asdf/core/tests/test_integration.py new file mode 100644 index 000000000..b808dc729 --- /dev/null +++ b/asdf/core/tests/test_integration.py @@ -0,0 +1,37 @@ +import pytest +import yaml + +import asdf +from asdf.core._integration import get_extensions, get_json_schema_resource_mappings + + +@pytest.mark.parametrize( + "uri", + [ + "http://json-schema.org/draft-04/schema", + ], +) +def test_get_resource_mappings(uri): + mappings = get_json_schema_resource_mappings() + + mapping = next(m for m in mappings if uri in m) + assert mapping is not None + + assert uri.encode("utf-8") in mapping[uri] + + +def test_get_extensions(): + extensions = get_extensions() + extension_uris = {e.extension_uri for e in extensions} + + # No duplicates + assert len(extension_uris) == len(extensions) + + resource_extension_uris = set() + resource_manager = asdf.get_config().resource_manager + for resource_uri in resource_manager: + if resource_uri.startswith("asdf://asdf-format.org/core/manifests/core-"): + resource_extension_uris.add(yaml.safe_load(resource_manager[resource_uri])["extension_uri"]) + + # Make sure every core manifest has a corresponding extension + assert resource_extension_uris == extension_uris diff --git a/asdf/exceptions.py b/asdf/exceptions.py index 7afbd8a01..eb47fcf44 100644 --- a/asdf/exceptions.py +++ b/asdf/exceptions.py @@ -1,3 +1,15 @@ +from jsonschema import ValidationError + +__all__ = [ + "AsdfConversionWarning", + "AsdfDeprecationWarning", + "AsdfProvisionalAPIWarning", + "AsdfWarning", + "DelimiterNotFoundError", + "ValidationError", +] + + class AsdfWarning(Warning): """ The base warning class from which all ASDF warnings should inherit. diff --git a/asdf/extension/__init__.py b/asdf/extension/__init__.py index f87b5673d..d26561e45 100644 --- a/asdf/extension/__init__.py +++ b/asdf/extension/__init__.py @@ -16,6 +16,7 @@ from ._manager import ExtensionManager, get_cached_extension_manager from ._manifest import ManifestExtension from ._tag import TagDefinition +from ._validator import Validator __all__ = [ # New API @@ -28,6 +29,7 @@ "Converter", "ConverterProxy", "Compressor", + "Validator", # Legacy API "AsdfExtension", "AsdfExtensionList", diff --git a/asdf/extension/_extension.py b/asdf/extension/_extension.py index 95397adfd..7cccd0be1 100644 --- a/asdf/extension/_extension.py +++ b/asdf/extension/_extension.py @@ -8,6 +8,7 @@ from ._converter import ConverterProxy from ._legacy import AsdfExtension from ._tag import TagDefinition +from ._validator import Validator class Extension(abc.ABC): @@ -117,6 +118,18 @@ def yaml_tag_handles(self): """ return {} + @property + def validators(self): + """ + Get the `asdf.extension.Validator` instances for additional + schema properties supported by this extension. + + Returns + ------- + iterable of asdf.extension.Validator + """ + return [] + class ExtensionProxy(Extension, AsdfExtension): """ @@ -193,6 +206,14 @@ def __init__(self, delegate, package_name=None, package_version=None): raise TypeError(msg) self._compressors.append(compressor) + self._validators = [] + if hasattr(self._delegate, "validators"): + for validator in self._delegate.validators: + if not isinstance(validator, Validator): + msg = "Extension property 'validators' must contain instances of asdf.extension.Validator" + raise TypeError(msg) + self._validators.append(validator) + @property def extension_uri(self): """ @@ -373,6 +394,18 @@ def yaml_tag_handles(self): """ return self._yaml_tag_handles + @property + def validators(self): + """ + Get the `asdf.extension.Validator` instances for additional + schema properties supported by this extension. + + Returns + ------- + list of asdf.extension.Validator + """ + return self._validators + def __eq__(self, other): if isinstance(other, ExtensionProxy): return other.delegate is self.delegate diff --git a/asdf/extension/_manager.py b/asdf/extension/_manager.py index b1ac5e17e..2274ce074 100644 --- a/asdf/extension/_manager.py +++ b/asdf/extension/_manager.py @@ -1,6 +1,7 @@ from functools import lru_cache -from asdf.util import get_class_name +from asdf.tagged import Tagged +from asdf.util import get_class_name, uri_match from ._extension import ExtensionProxy @@ -25,6 +26,8 @@ def __init__(self, extensions): # This dict has both str and type keys: self._converters_by_type = {} + validators = set() + for extension in self._extensions: for tag_def in extension.tags: if tag_def.tag_uri not in self._tag_defs_by_tag: @@ -47,6 +50,10 @@ def __init__(self, extensions): self._converters_by_type[typ] = converter self._converters_by_type[type_class_name] = converter + validators.update(extension.validators) + + self._validator_manager = ValidatorManager(validators) + @property def extensions(self): """ @@ -182,6 +189,10 @@ def get_converter_for_type(self, typ): ) raise KeyError(msg) from None + @property + def validator_manager(self): + return self._validator_manager + def get_cached_extension_manager(extensions): """ @@ -214,3 +225,80 @@ def get_cached_extension_manager(extensions): @lru_cache def _get_cached_extension_manager(extensions): return ExtensionManager(extensions) + + +class ValidatorManager: + """ + Wraps a list of validators and indexes them by schema property. + + Parameters + ---------- + validators : iterable of asdf.extension.Validator + List of validators to manage. + """ + + def __init__(self, validators): + self._validators = list(validators) + + self._validators_by_schema_property = {} + for validator in self._validators: + if validator.schema_property not in self._validators_by_schema_property: + self._validators_by_schema_property[validator.schema_property] = set() + self._validators_by_schema_property[validator.schema_property].add(validator) + + def validate(self, schema_property, schema_property_value, node, schema): + """ + Validate an ASDF tree node against a schema property. + + Parameters + ---------- + schema_property : str + Name of the schema property (identifies the validator(s) to use). + schema_property_value : object + Value of the schema property. + node : asdf.tagged.Tagged + The ASDF node to validate. + schema : dict + The schema object that contains the property that triggered + the validation. + + Yields + ------ + asdf.exceptions.ValidationError + """ + if schema_property in self._validators_by_schema_property: + for validator in self._validators_by_schema_property[schema_property]: + if _validator_matches(validator, node): + yield from validator.validate(schema_property_value, node, schema) + + def get_jsonschema_validators(self): + """ + Get a dictionary of validator methods suitable for use + with the jsonschema library. + + Returns + ------- + dict of str: callable + """ + result = {} + + for schema_property in self._validators_by_schema_property: + result[schema_property] = self._get_jsonschema_validator(schema_property) + + return result + + def _get_jsonschema_validator(self, schema_property): + def _validator(_, schema_property_value, node, schema): + return self.validate(schema_property, schema_property_value, node, schema) + + return _validator + + +def _validator_matches(validator, node): + if any(t == "**" for t in validator.tags): + return True + + if not isinstance(node, Tagged): + return False + + return any(uri_match(t, node._tag) for t in validator.tags) diff --git a/asdf/extension/_manifest.py b/asdf/extension/_manifest.py index dbfd2548e..43b5b3610 100644 --- a/asdf/extension/_manifest.py +++ b/asdf/extension/_manifest.py @@ -19,6 +19,9 @@ class ManifestExtension(Extension): compressors : iterable of asdf.extension.Compressor, optional Compressor instances to support additional binary block compression options. + validators : iterable of asdf.extension.Validator, optional + Validator instances to support validation of custom + schema properties. legacy_class_names : iterable of str, optional Fully-qualified class names used by older versions of this extension. @@ -43,7 +46,7 @@ def from_uri(cls, manifest_uri, **kwargs): manifest = yaml.safe_load(get_config().resource_manager[manifest_uri]) return cls(manifest, **kwargs) - def __init__(self, manifest, *, legacy_class_names=None, converters=None, compressors=None): + def __init__(self, manifest, *, legacy_class_names=None, converters=None, compressors=None, validators=None): self._manifest = manifest if legacy_class_names is None: @@ -61,6 +64,11 @@ def __init__(self, manifest, *, legacy_class_names=None, converters=None, compre else: self._compressors = compressors + if validators is None: + self._validators = [] + else: + self._validators = validators + @property def extension_uri(self): return self._manifest["extension_uri"] @@ -93,6 +101,10 @@ def converters(self): def compressors(self): return self._compressors + @property + def validators(self): + return self._validators + @property def tags(self): result = [] diff --git a/asdf/extension/_validator.py b/asdf/extension/_validator.py new file mode 100644 index 000000000..c695ac7a9 --- /dev/null +++ b/asdf/extension/_validator.py @@ -0,0 +1,56 @@ +import abc + + +class Validator(abc.ABC): + """ + Abstract base class for plugins that handle custom validators + in ASDF schemas. + """ + + @abc.abstractproperty + def schema_property(self): + """ + Name of the schema property used to invoke this validator. + """ + + @abc.abstractproperty + def tags(self): + """ + Get the YAML tags that are appropriate to this validator. + URI patterns are permitted, see `asdf.util.uri_match` for details. + + Returns + ------- + iterable of str + Tag URIs or URI patterns. + """ + + @abc.abstractmethod + def validate(self, schema_property_value, node, schema): + """ + Validate the given node from the ASDF tree. + + Parameters + ---------- + schema_property_value : object + The value assigned to the schema property associated with this + valdiator. + + node : asdf.tagged.Tagged + A tagged node from the tree. Guaranteed to bear a tag that + matches one of the URIs returned by this validator's tags property. + + schema : dict + The schema object that contains the property that triggered + the validation. Typically implementations of this method do + not need to make use of this object, but sometimes the behavior + of a validator depends on other schema properties. An example is + the built-in "additionalProperties" property, which needs to know + the contents of the "properties" property in order to determine + which node properties are additional. + + Yields + ------ + asdf.exceptions.ValidationError + Yield an instance of ValidationError for each error present in the node. + """ diff --git a/asdf/resource.py b/asdf/resource.py index 0cf45736e..646ee29a9 100644 --- a/asdf/resource.py +++ b/asdf/resource.py @@ -14,7 +14,6 @@ "DirectoryResourceMapping", "ResourceManager", "JsonschemaResourceMapping", - "get_json_schema_resource_mappings", ] @@ -192,9 +191,3 @@ def __iter__(self): def __repr__(self): return "JsonschemaResourceMapping()" - - -def get_json_schema_resource_mappings(): - return [ - JsonschemaResourceMapping(), - ] diff --git a/asdf/schema.py b/asdf/schema.py index d8e2ae81e..ec703e8f5 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -584,6 +584,7 @@ def get_validator( if validators is None: validators = util.HashableDict(YAML_VALIDATORS.copy()) validators.update(ctx.extension_list.validators) + validators.update(ctx.extension_manager.validator_manager.get_jsonschema_validators()) kwargs["resolver"] = _make_resolver(url_mapping) diff --git a/asdf/tags/core/ndarray.py b/asdf/tags/core/ndarray.py index 33606b7b7..62801091a 100644 --- a/asdf/tags/core/ndarray.py +++ b/asdf/tags/core/ndarray.py @@ -577,9 +577,7 @@ def operation(self, *args): return operation -classes_to_modify = NDArrayType.__versioned_siblings + [ - NDArrayType, -] +classes_to_modify = [*NDArrayType.__versioned_siblings, NDArrayType] for op in [ "__neg__", "__pos__", @@ -735,6 +733,3 @@ def validate_datatype(validator, datatype, instance, schema): f"Expected {numpy_dtype_to_asdf_datatype(out_type)[0]}, " f"got {numpy_dtype_to_asdf_datatype(in_type)[0]}", ) - - -NDArrayType.validators = {"ndim": validate_ndim, "max_ndim": validate_max_ndim, "datatype": validate_datatype} diff --git a/asdf/tests/test_config.py b/asdf/tests/test_config.py index 582b393dd..8a328a043 100644 --- a/asdf/tests/test_config.py +++ b/asdf/tests/test_config.py @@ -4,7 +4,8 @@ import pytest import asdf -from asdf import get_config, resource +from asdf import get_config +from asdf.core._integration import get_json_schema_resource_mappings from asdf.extension import BuiltinExtension, ExtensionProxy from asdf.resource import ResourceMappingProxy @@ -110,7 +111,7 @@ def test_array_inline_threshold(): def test_resource_mappings(): with asdf.config_context() as config: - core_mappings = resource.get_json_schema_resource_mappings() + asdf_standard.integration.get_resource_mappings() + core_mappings = get_json_schema_resource_mappings() + asdf_standard.integration.get_resource_mappings() default_mappings = config.resource_mappings assert len(default_mappings) >= len(core_mappings) diff --git a/asdf/tests/test_extension.py b/asdf/tests/test_extension.py index 45e7edb0b..846b4ff1f 100644 --- a/asdf/tests/test_extension.py +++ b/asdf/tests/test_extension.py @@ -1,8 +1,8 @@ import pytest from packaging.specifiers import SpecifierSet -from asdf import config_context -from asdf.exceptions import AsdfDeprecationWarning +from asdf import AsdfFile, config_context +from asdf.exceptions import AsdfDeprecationWarning, ValidationError from asdf.extension import ( AsdfExtension, BuiltinExtension, @@ -14,6 +14,7 @@ ExtensionProxy, ManifestExtension, TagDefinition, + Validator, get_cached_asdf_extension_list, get_cached_extension_manager, ) @@ -53,14 +54,16 @@ def __init__( self, converters=None, compressors=None, + validators=None, asdf_standard_requirement=None, tags=None, legacy_class_names=None, ): self._converters = [] if converters is None else converters self._compressors = [] if compressors is None else compressors + self._validators = [] if validators is None else validators self._asdf_standard_requirement = asdf_standard_requirement - self._tags = tags + self._tags = [] if tags is None else tags self._legacy_class_names = [] if legacy_class_names is None else legacy_class_names @property @@ -71,6 +74,10 @@ def converters(self): def compressors(self): return self._compressors + @property + def validators(self): + return self._validators + @property def asdf_standard_requirement(self): return self._asdf_standard_requirement @@ -126,6 +133,15 @@ def label(self): return b"mini" +class MinimalValidator(Validator): + schema_property = "fail" + tags = ["**"] + + def validate(self, fail, node, schema): + if fail: + yield ValidationError("Node was doomed to fail") + + # Some dummy types for testing converters: class FooType: pass @@ -162,6 +178,7 @@ def test_extension_proxy(): assert proxy.asdf_standard_requirement == SpecifierSet() assert proxy.converters == [] assert proxy.compressors == [] + assert proxy.validators == [] assert proxy.tags == [] assert proxy.types == [] assert proxy.tag_mapping == [] @@ -180,6 +197,7 @@ def test_extension_proxy(): assert subclassed_proxy.asdf_standard_requirement == proxy.asdf_standard_requirement assert subclassed_proxy.converters == proxy.converters assert subclassed_proxy.compressors == proxy.compressors + assert subclassed_proxy.validators == proxy.validators assert subclassed_proxy.tags == proxy.tags assert subclassed_proxy.types == proxy.types assert subclassed_proxy.tag_mapping == proxy.tag_mapping @@ -193,9 +211,11 @@ def test_extension_proxy(): # Test with all properties present: converters = [MinimumConverter(tags=["asdf://somewhere.org/extensions/full/tags/foo-*"], types=[])] compressors = [MinimalCompressor()] + validators = [MinimalValidator()] extension = FullExtension( converters=converters, compressors=compressors, + validators=validators, asdf_standard_requirement=">=1.4.0", tags=["asdf://somewhere.org/extensions/full/tags/foo-1.0"], legacy_class_names=["foo.extensions.SomeOldExtensionClass"], @@ -207,6 +227,7 @@ def test_extension_proxy(): assert proxy.asdf_standard_requirement == SpecifierSet(">=1.4.0") assert proxy.converters == [ConverterProxy(c, proxy) for c in converters] assert proxy.compressors == compressors + assert proxy.validators == validators assert len(proxy.tags) == 1 assert proxy.tags[0].tag_uri == "asdf://somewhere.org/extensions/full/tags/foo-1.0" assert proxy.types == [] @@ -230,7 +251,11 @@ def test_extension_proxy(): with pytest.raises(TypeError): ExtensionProxy(FullExtension(compressors=[object()])) - # Unparsable ASDF Standard requirement: + # Should fail with a bad validator + with pytest.raises(TypeError): + ExtensionProxy(FullExtension(validators=[object()])) + + # Unparseable ASDF Standard requirement: with pytest.raises(ValueError): ExtensionProxy(FullExtension(asdf_standard_requirement="asdf-standard >= 1.4.0")) @@ -597,6 +622,7 @@ def test_manifest_extension(): assert extension.asdf_standard_requirement is None assert extension.converters == [] assert extension.compressors == [] + assert extension.validators == [] assert extension.tags == [] proxy = ExtensionProxy(extension) @@ -605,6 +631,7 @@ def test_manifest_extension(): assert proxy.asdf_standard_requirement == SpecifierSet() assert proxy.converters == [] assert proxy.compressors == [] + assert proxy.validators == [] assert proxy.tags == [] with config_context() as config: @@ -639,6 +666,7 @@ def from_yaml_tree(self, *args): pass converter = FooConverter() + validator = MinimalValidator() compressor = MinimalCompressor() extension = ManifestExtension.from_uri( @@ -646,12 +674,14 @@ def from_yaml_tree(self, *args): legacy_class_names=["foo.extension.LegacyExtension"], converters=[converter], compressors=[compressor], + validators=[validator], ) assert extension.extension_uri == "asdf://somewhere.org/extensions/foo" assert extension.legacy_class_names == ["foo.extension.LegacyExtension"] assert extension.asdf_standard_requirement == SpecifierSet(">=1.6.0,<2.0.0") assert extension.converters == [converter] assert extension.compressors == [compressor] + assert extension.validators == [validator] assert len(extension.tags) == 2 assert extension.tags[0] == "asdf://somewhere.org/tags/bar" assert extension.tags[1].tag_uri == "asdf://somewhere.org/tags/baz" @@ -665,6 +695,7 @@ def from_yaml_tree(self, *args): assert proxy.asdf_standard_requirement == SpecifierSet(">=1.6.0,<2.0.0") assert proxy.converters == [ConverterProxy(converter, proxy)] assert proxy.compressors == [compressor] + assert proxy.validators == [validator] assert len(proxy.tags) == 2 assert proxy.tags[0].tag_uri == "asdf://somewhere.org/tags/bar" assert proxy.tags[1].tag_uri == "asdf://somewhere.org/tags/baz" @@ -686,3 +717,42 @@ def from_yaml_tree(self, *args): proxy = ExtensionProxy(extension) assert proxy.asdf_standard_requirement == SpecifierSet("==1.6.0") + + +def test_validator(): + validator = MinimalValidator() + extension = FullExtension(validators=[validator]) + + failing_schema = """ + type: object + properties: + foo: + fail: true + """ + + passing_schema = """ + type: object + properties: + foo: + fail: false + """ # noqa: S105 + + with config_context() as config: + config.add_extension(extension) + config.add_resource_mapping( + { + "asdf://somewhere.org/schemas/failing": failing_schema, + "asdf://somewhere.org/schemas/passing": passing_schema, + }, + ) + + with AsdfFile(custom_schema="asdf://somewhere.org/schemas/passing") as af: + af["foo"] = "bar" + af.validate() + + with AsdfFile(custom_schema="asdf://somewhere.org/schemas/failing") as af: + af.validate() + + af["foo"] = "bar" + with pytest.raises(ValidationError): + af.validate() diff --git a/asdf/tests/test_resource.py b/asdf/tests/test_resource.py index 483ba3547..30775b560 100644 --- a/asdf/tests/test_resource.py +++ b/asdf/tests/test_resource.py @@ -2,12 +2,7 @@ import pytest -from asdf.resource import ( - JsonschemaResourceMapping, - ResourceManager, - ResourceMappingProxy, - get_json_schema_resource_mappings, -) +from asdf.resource import JsonschemaResourceMapping, ResourceManager, ResourceMappingProxy def test_resource_manager(): @@ -59,21 +54,6 @@ def test_jsonschema_resource_mapping(): assert repr(mapping) == "JsonschemaResourceMapping()" -@pytest.mark.parametrize( - "uri", - [ - "http://json-schema.org/draft-04/schema", - ], -) -def test_get_json_schema_resource_mappings(uri): - mappings = get_json_schema_resource_mappings() - - mapping = next(m for m in mappings if uri in m) - assert mapping is not None - - assert uri.encode("utf-8") in mapping[uri] - - def test_proxy_is_mapping(): assert isinstance(ResourceMappingProxy({}), Mapping) diff --git a/docs/asdf/extending/extensions.rst b/docs/asdf/extending/extensions.rst index af7d51129..900be1a53 100644 --- a/docs/asdf/extending/extensions.rst +++ b/docs/asdf/extending/extensions.rst @@ -7,8 +7,8 @@ Extensions ========== An ASDF "extension" is a supplement to the core ASDF specification that -describes additional YAML tags or binary block compressors which -may be used when writing files. In this library, extensions implement the +describes additional YAML tags, binary block compressors, or schema validators which +may be used when reading and writing files. In this library, extensions implement the `Extension` interface and can be installed manually by the user or automatically by a package using Python's entry points mechanism. @@ -36,8 +36,8 @@ provide its URI as a property: Note that this is an "empty" extension that does not extend the library in any meaningful way; other attributes must be implemented to actually -support additional tags and/or compressors. Read on for a description of the rest -of the Extension interface. +support additional tags, compressors and/or validators. Read on for a description +of the rest of the Extension interface. Additional tags --------------- @@ -154,6 +154,26 @@ Tag handles can be defined in the ``yaml_tag_handles`` property of an extension: extension_uri = "asdf://example.com/example-project/extensions/foo-1.0.0" yaml_tag_handles = {"!example!": "asdf://example.com/example-project/tags/"} +Additional schema validators +---------------------------- + +Schema validators implement the `Validator` interface +and are included in an extension via the ``validators`` property: + +.. code-block:: python + + from asdf.extension import Extension, Validator + + + class FooValidator(Validator): + # ... + pass + + + class FooExtension(Extension): + extension_uri = "asdf://example.com/example-project/extensions/foo-1.0.0" + validators = [FooValidator()] + ASDF Standard version requirement --------------------------------- diff --git a/docs/asdf/extending/use_cases.rst b/docs/asdf/extending/use_cases.rst index 9747f3d79..63f1be0dd 100644 --- a/docs/asdf/extending/use_cases.rst +++ b/docs/asdf/extending/use_cases.rst @@ -170,3 +170,28 @@ that in an extension. Now the compression algorithm will be available for both reading and writing ASDF files. Users writing files will simply need to specify the new 4-byte compression code when making calls to `asdf.AsdfFile.set_array_compression`. + +Support a new schema property +============================= + +In order to support custom validation behavior that is triggered by a new schema +property, we need to implement the `~asdf.extension.Validator` interface +and install that in an extension. + +1. Determine the tag or tags that this property will apply to. + +2. Select a schema property for the validator. The property need not be globally unique, + but it should be unique among the validators that apply to the tag(s), and must + not collide with any of the built-in JSON schema properties (type, additionalProperties, etc). + +3. Implement a `~asdf.extension.Validator` class that associates the schema property and tag(s) + with a ``validate`` method that does the work of checking an ADSF node against the schema. + +4. Implement an `~asdf.extension.Extension` class which is the vehicle + for plugging our validator into the `asdf` library. See :ref:`extending_extensions` + for a discussion of the Extension interface. + +5. Install the extension via one of the two available methods. See + :ref:`extending_extensions_installing` for instructions. + +Now the new schema property will have an effect when validating ASDF files. diff --git a/docs/asdf/extending/validators.rst b/docs/asdf/extending/validators.rst new file mode 100644 index 000000000..49f0ef6ec --- /dev/null +++ b/docs/asdf/extending/validators.rst @@ -0,0 +1,79 @@ +.. currentmodule:: asdf.extension + +.. _extending_validators: + +========== +Validators +========== + +The `~asdf.extension.Validator` interface provides support for a custom +ASDF schema property. The Validator identifies the schema property and +tag(s) that it works on and provides a method for doing the work of +validation. + +The Validator interface +======================= + +Every Validator implementation must provide two required properties and +one required method: + +`Validator.schema_property` - The schema property that triggers this +validator. The property need not be globally unique, but it should be +unique among the validators that apply to the tag(s), and must not +collide with any of the built-in JSON schema propeties (type, additionalProperties, +etc). + +`Validator.tags` - a list of tag URIs or URI patterns handled by the validator. +Patterns may include the wildcard character ``*``, which matches any sequence of +characters up to a ``/``, or ``**``, which matches any sequence of characters. +The `~asdf.util.uri_match` method can be used to test URI patterns. + +`Validator.validate` - a method that accepts the schema property value, a tagged ASDF node, +and the surrounding schema dict, and performs validation on the node. For every error +present, the method should yield an instance of ``asdf.exceptions.ValidationError``. + +A simple example +================ + +Say we have a custom tagged object, ``asdf://example.com/example-project/tags/rectangle-1.0.0``, +which describes a rectangle with ``width`` and ``height`` properties. Let's implement +a validator that checks that the area of the rectangle is less than some maximum value. + +The schema property will be called ``max_area``, so our validator will look like this: + +.. code-block:: python + + from asdf.extension import Validator + from asdf.exceptions import ValidationError + + class MaxAreaValidator(Validator): + schema_property = "max_area" + tags = ["asdf://example.com/example-project/tags/rectangle-1.0.0"] + + def validate(self, max_area, node, schema): + area = node["width"] * node["height"] + if area > max_area: + yield ValidationError( + f"Rectangle with area {area} exceeds max area of {max_area}" + ) + +Note that the validator operates on raw ASDF tagged nodes, and not the custom +Python object that they'll be converted to. + +In order to use this Validator, we'll need to create a simple extension +around it and install that extension: + +.. code-block:: python + + import asdf + from asdf.extension import Extension + + class ShapesExtension(Extension): + extension_uri = "asdf://example.com/shapes/extensions/shapes-1.0.0" + validators = [MaxAreaValidator()] + tags = ["asdf://example.com/shapes/tags/rectangle-1.0.0"] + + asdf.get_config().add_extension(ShapesExtension()) + +Now we can include a ``max_area`` property in a schema and have it +restrict the area of a rectangle. diff --git a/docs/index.rst b/docs/index.rst index 8e199581b..d1df3acf8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -52,6 +52,7 @@ Extending ASDF asdf/extending/extensions asdf/extending/manifests asdf/extending/compressors + asdf/extending/validators asdf/extending/legacy API Documentation diff --git a/pyproject.toml b/pyproject.toml index f6adf04c0..8be5f3eff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,8 @@ tests = [ 'tracker' = 'https://github.com/asdf-format/asdf/issues' [project.entry-points] -'asdf.resource_mappings' = {asdf = 'asdf.resource:get_json_schema_resource_mappings'} +'asdf.resource_mappings' = {asdf = 'asdf.core._integration:get_json_schema_resource_mappings'} +'asdf.extensions' = {asdf = 'asdf.core._integration:get_extensions'} asdf_extensions = {builtin = 'asdf.extension:BuiltinExtension'} console_scripts = {asdftool = 'asdf.commands.main:main'} pytest11 = {asdf_schema_tester = 'pytest_asdf.plugin'}