Add config spec data model consumer

.azure-pipelines/templates/run-validations.yml

@@ -33,6 +33,11 @@ steps:
     ddev validate config
   displayName: 'Validate default configuration files'
 
+- script: |
+    echo "ddev validate models"
+    ddev validate models
+  displayName: 'Validate configuration data models'
+
 - script: |
     echo "ddev validate dashboards"
     ddev validate dashboards

.flake8

@@ -8,3 +8,14 @@ ignore = E203,E722,E741,W503,G200
 exclude = .eggs,.tox,build,compat.py,__init__.py,datadog_checks_dev/datadog_checks/dev/tooling/templates/*,*/datadog_checks/*/vendor/*
 max-line-length = 120
 enable-extensions=G
+per-file-ignores =
+    # potentially long literal strings
+    datadog_checks/*/config_models/deprecations.py: E501
+    tests/models/config_models/deprecations.py: E501
+    # https://pydantic-docs.helpmanual.io/usage/validators/
+    # > validators are "class methods", so the first argument value they
+    #   receive is the UserModel class, not an instance of UserModel
+    datadog_checks/*/config_models/instance.py: B902
+    datadog_checks/*/config_models/shared.py: B902
+    tests/models/config_models/instance.py: B902
+    tests/models/config_models/shared.py: B902

datadog_checks_base/datadog_checks/base/checks/base.py

@@ -15,10 +15,11 @@
 from typing import TYPE_CHECKING, Any, AnyStr, Callable, Deque, Dict, List, Optional, Sequence, Tuple, Union
 
 import yaml
-from six import binary_type, iteritems, text_type
+from six import PY2, binary_type, iteritems, raise_from, text_type
 
 from ..config import is_affirmative
 from ..constants import ServiceCheck
+from ..errors import ConfigurationError
 from ..types import (
     AgentConfigType,
     Event,
@@ -63,6 +64,9 @@
 
     monkey_patch_pyyaml()
 
+if not PY2:
+    from pydantic import ValidationError
+
 if TYPE_CHECKING:
     import ssl
 
@@ -241,9 +245,16 @@ def __init__(self, *args, **kwargs):
         # Setup metric limits
         self.metric_limiter = self._get_metric_limiter(self.name, instance=self.instance)
 
+        # Lazily load and validate config
+        self._config_model_instance = None  # type: Any
+        self._config_model_shared = None  # type: Any
+
         # Functions that will be called exactly once (if successful) before the first check run
         self.check_initializations = deque([self.send_config_metadata])  # type: Deque[Callable[[], None]]
 
+        if not PY2:
+            self.check_initializations.append(self.load_configuration_models)
+
     def _get_metric_limiter(self, name, instance=None):
         # type: (str, InstanceType) -> Optional[Limiter]
         limit = self._get_metric_limit(instance=instance)
@@ -364,6 +375,79 @@ def in_developer_mode(self):
         self._log_deprecation('in_developer_mode')
         return False
 
+    def load_configuration_models(self):
+        # 'datadog_checks.<PACKAGE>.<MODULE>...'
+        module_parts = self.__module__.split('.')
+        package_path = '{}.config_models'.format('.'.join(module_parts[:2]))
+
+        if self._config_model_shared is None:
+            raw_shared_config = self._get_config_model_initialization_data()
+            raw_shared_config.update(self._get_shared_config())
+
+            shared_config = self.load_configuration_model(package_path, 'SharedConfig', raw_shared_config)
+            if shared_config is not None:
+                self._config_model_shared = shared_config
+
+        if self._config_model_instance is None:
+            raw_instance_config = self._get_config_model_initialization_data()
+            raw_instance_config.update(self._get_instance_config())
+
+            instance_config = self.load_configuration_model(package_path, 'InstanceConfig', raw_instance_config)
+            if instance_config is not None:
+                self._config_model_instance = instance_config
+
+    @staticmethod
+    def load_configuration_model(import_path, model_name, config):
+        try:
+            package = importlib.import_module(import_path)
+        # TODO: remove the type ignore when we drop Python 2
+        except ModuleNotFoundError as e:  # type: ignore
+            # Don't fail if there are no models
+            if str(e).startswith('No module named '):
+                return
+
+            raise
+
+        model = getattr(package, model_name, None)
+        if model is not None:
+            try:
+                config_model = model(**config)
+            # TODO: remove the type ignore when we drop Python 2
+            except ValidationError as e:  # type: ignore
+                errors = e.errors()
+                num_errors = len(errors)
+                message_lines = [
+                    'Detected {} error{} while loading configuration model `{}`:'.format(
+                        num_errors, 's' if num_errors > 1 else '', model_name
+                    )
+                ]
+
+                for error in errors:
+                    message_lines.append(
+                        ' -> '.join(
+                            # Start array indexes at one for user-friendliness
+                            str(loc + 1) if isinstance(loc, int) else str(loc)
+                            for loc in error['loc']
+                        )
+                    )
+                    message_lines.append('  {}'.format(error['msg']))
+
+                raise_from(ConfigurationError('\n'.join(message_lines)), None)
+            else:
+                return config_model
+
+    def _get_shared_config(self):
+        # Any extra fields will be available during a config model's initial validation stage
+        return copy.deepcopy(self.init_config)
+
+    def _get_instance_config(self):
+        # Any extra fields will be available during a config model's initial validation stage
+        return copy.deepcopy(self.instance)
+
+    def _get_config_model_initialization_data(self):
+        # Allow for advanced functionality during the initial root validation stage
+        return {'__data': {'logger': self.log, 'warning': self.warning}}
+
     def register_secret(self, secret):
         # type: (str) -> None
         """

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -19,6 +19,7 @@ flup-py3==1.0.3; python_version > "3.0"
 flup==1.0.3.dev-20110405; python_version < "3.0"
 futures==3.3.0; python_version < "3.0"
 gearman==2.0.2; sys_platform != "win32" and python_version < "3.0"
+immutables==0.15; python_version > "3.0"
 in-toto==0.5.0
 ipaddress==1.0.22; python_version < "3.0"
 jaydebeapi==1.2.3
@@ -42,6 +43,7 @@ psutil==5.7.2
 psycopg2-binary==2.8.4
 pyasn1==0.4.6
 pycryptodomex==3.9.4
+pydantic==1.8; python_version > "3.0"
 pyhdb==0.3.4
 pyjwt==1.7.1
 pymongo==3.8.0

datadog_checks_base/datadog_checks/base/utils/models/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

datadog_checks_base/datadog_checks/base/utils/models/fields.py

@@ -0,0 +1,17 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from pydantic.fields import SHAPE_MAPPING, SHAPE_SEQUENCE, SHAPE_SINGLETON
+
+
+def get_default_field_value(field, value):
+    if field.shape == SHAPE_MAPPING:
+        return {}
+    elif field.shape == SHAPE_SEQUENCE:
+        return []
+    elif field.shape == SHAPE_SINGLETON:
+        field_type = field.type_
+        if field_type in (float, int, str):
+            return field_type()
+
+    return value

datadog_checks_base/datadog_checks/base/utils/models/types.py

@@ -0,0 +1,16 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from collections.abc import Mapping, Sequence
+
+from immutables import Map
+
+
+def make_immutable_check_config(obj):
+    if isinstance(obj, Sequence) and not isinstance(obj, str):
+        return tuple(make_immutable_check_config(item) for item in obj)
+    elif isinstance(obj, Mapping):
+        # There are no ordering guarantees, see https://github.com/MagicStack/immutables/issues/57
+        return Map((k, make_immutable_check_config(v)) for k, v in obj.items())
+
+    return obj

datadog_checks_base/datadog_checks/base/utils/models/validation/__init__.py

@@ -0,0 +1,4 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from . import core

datadog_checks_base/datadog_checks/base/utils/models/validation/core.py

@@ -0,0 +1,21 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from ..types import make_immutable_check_config
+
+
+def initialize_config(values, **kwargs):
+    # This is what is returned by the initial root validator of each config model.
+    return values
+
+
+def finalize_config(values, **kwargs):
+    # This is what is returned by the final root validator of each config model. Note:
+    #
+    # 1. the final object must be a dict
+    # 2. we maintain the original order of keys
+    return {field: make_immutable_check_config(value) for field, value in values.items()}
+
+
+def handle_deprecations(values, model_id, deprecations):
+    return values

datadog_checks_base/requirements.in

@@ -5,12 +5,14 @@ contextlib2==0.6.0; python_version < '3.0'
 cryptography==3.3.2
 ddtrace==0.32.2
 enum34==1.1.6; python_version < '3.0'
+immutables==0.15; python_version > '3.0'
 ipaddress==1.0.22; python_version < '3.0'
 kubernetes==8.0.1
 mmh3==2.5.1
 orjson==2.6.1; python_version > '3.0'
 prometheus-client==0.9.0
 protobuf==3.7.0
+pydantic==1.8; python_version > '3.0'
 pyjwt==1.7.1
 pysocks==1.7.0
 python-dateutil==2.8.0

datadog_checks_base/tests/models/__init__.py

@@ -0,0 +1,3 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)

datadog_checks_base/tests/models/config_models/__init__.py

@@ -0,0 +1,18 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from .instance import InstanceConfig
+from .shared import SharedConfig
+
+
+class ConfigProvider:
+    _config_model_instance: InstanceConfig
+    _config_model_shared: SharedConfig
+
+    @property
+    def config(self) -> InstanceConfig:
+        return self._config_model_instance
+
+    @property
+    def shared_config(self) -> SharedConfig:
+        return self._config_model_shared

datadog_checks_base/tests/models/config_models/defaults.py

@@ -0,0 +1,52 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from datadog_checks.base.utils.models.fields import get_default_field_value
+
+
+def shared_deprecated(field, value):
+    return get_default_field_value(field, value)
+
+
+def shared_timeout(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_array(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_deprecated(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_flag(field, value):
+    return False
+
+
+def instance_hyphenated_name(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_mapping(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_obj(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_pass_(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_pid(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_text(field, value):
+    return get_default_field_value(field, value)
+
+
+def instance_timeout(field, value):
+    return get_default_field_value(field, value)

datadog_checks_base/tests/models/config_models/deprecations.py

@@ -0,0 +1,11 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+
+def shared():
+    return {'deprecated': {'Release': '8.0.0', 'Migration': 'do this\nand that\n'}}
+
+
+def instance():
+    return {'deprecated': {'Release': '9.0.0', 'Migration': 'do this\nand that\n'}}

datadog_checks_base/tests/models/config_models/instance.py

@@ -0,0 +1,63 @@
+# (C) Datadog, Inc. 2021-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from __future__ import annotations
+
+from typing import Any, Mapping, Optional, Sequence
+
+from pydantic import BaseModel, Field, root_validator, validator
+
+from datadog_checks.base.utils.functions import identity
+from datadog_checks.base.utils.models import validation
+
+from . import defaults, deprecations, validators
+
+
+class Obj(BaseModel):
+    class Config:
+        allow_mutation = False
+
+    bar: Optional[Sequence[str]]
+    foo: bool
+
+
+class InstanceConfig(BaseModel):
+    class Config:
+        allow_mutation = False
+
+    array: Optional[Sequence[str]]
+    deprecated: Optional[str]
+    flag: Optional[bool]
+    hyphenated_name: Optional[str] = Field(None, alias='hyphenated-name')
+    mapping: Optional[Mapping[str, Any]]
+    obj: Optional[Obj]
+    pass_: Optional[str] = Field(None, alias='pass')
+    pid: Optional[int]
+    text: Optional[str]
+    timeout: Optional[float]
+
+    @root_validator(pre=True)
+    def _handle_deprecations(cls, values):
+        return validation.core.handle_deprecations(values, 'instance', deprecations.instance())
+
+    @root_validator(pre=True)
+    def _initial_validation(cls, values):
+        return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values))
+
+    @validator('*', pre=True, always=True)
+    def _ensure_defaults(cls, v, field):
+        if v is not None or field.required:
+            return v
+
+        return getattr(defaults, f'instance_{field.name}')(field, v)
+
+    @validator('*')
+    def _run_validations(cls, v, field):
+        if not v:
+            return v
+
+        return getattr(validators, f'instance_{field.name}', identity)(v, field=field)
+
+    @root_validator(pre=False)
+    def _final_validation(cls, values):
+        return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values))