ADAP-2: Materialized Views

.changes/unreleased/Features-20230329-120313.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add support for materialized views
+time: 2023-03-29T12:03:13.862041-04:00
+custom:
+  Author: mikealfare McKnight-42
+  Issue: "6911"

core/dbt/adapters/base/relation.py

@@ -328,6 +328,10 @@ def is_cte(self) -> bool:
     def is_view(self) -> bool:
         return self.type == RelationType.View
 
+    @property
+    def is_materialized_view(self) -> bool:
+        return self.type == RelationType.MaterializedView
+
     @classproperty
     def Table(cls) -> str:
         return str(RelationType.Table)
@@ -344,6 +348,10 @@ def View(cls) -> str:
     def External(cls) -> str:
         return str(RelationType.External)
 
+    @classproperty
+    def MaterializedView(cls) -> str:
+        return str(RelationType.MaterializedView)
+
     @classproperty
     def get_relation_type(cls) -> Type[RelationType]:
         return RelationType

core/dbt/adapters/relation_configs/README.md

@@ -0,0 +1,25 @@
+# RelationConfig
+This package serves as an initial abstraction for managing the inspection of existing relations and determining
+changes on those relations. It arose from the materialized view work and is currently only supporting 
+materialized views for Postgres and Redshift as well as dynamic tables for Snowflake. There are three main
+classes in this package.
+
+## RelationConfigBase
+This is a very small class that only has a `from_dict()` method and a default `NotImplementedError()`. At some
+point this could be replaced by a more robust framework, like `mashumaro` or `pydantic`.
+
+## RelationConfigChange
+This class inherits from `RelationConfigBase` ; however, this can be thought of as a separate class. The subclassing
+merely points to the idea that both classes would likely inherit from the same class in a `mashumaro` or
+`pydantic` implementation. This class is much more restricted in attribution. It should really only
+ever need an `action` and a `context`. This can be though of as being analogous to a web request. You need to
+know what you're doing (`action`: 'create' = GET, 'drop' = DELETE, etc.) and the information (`context`) needed
+to make the change. In our scenarios, the context tends to be an instance of `RelationConfigBase` corresponding
+to the new state.
+
+## RelationConfigValidationMixin
+This mixin provides optional validation mechanics that can be applied to either `RelationConfigBase` or
+`RelationConfigChange` subclasses. A validation rule is a combination of a `validation_check`, something
+that should evaluate to `True`, and an optional `validation_error`, an instance of `DbtRuntimeError`
+that should be raised in the event the `validation_check` fails. While optional, it's recommended that
+the `validation_error` be provided for clearer transparency to the end user.

core/dbt/adapters/relation_configs/__init__.py

@@ -0,0 +1,12 @@
+from dbt.adapters.relation_configs.config_base import (  # noqa: F401
+    RelationConfigBase,
+    RelationResults,
+)
+from dbt.adapters.relation_configs.config_change import (  # noqa: F401
+    RelationConfigChangeAction,
+    RelationConfigChange,
+)
+from dbt.adapters.relation_configs.config_validation import (  # noqa: F401
+    RelationConfigValidationMixin,
+    RelationConfigValidationRule,
+)

core/dbt/adapters/relation_configs/config_base.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+from typing import Union, Dict
+
+import agate
+from dbt.utils import filter_null_values
+
+
+"""
+This is what relation metadata from the database looks like. It's a dictionary because there will be
+multiple grains of data for a single object. For example, a materialized view in Postgres has base level information,
+like name. But it also can have multiple indexes, which needs to be a separate query. It might look like this:
+
+{
+    "base": agate.Row({"table_name": "table_abc", "query": "select * from table_def"})
+    "indexes": agate.Table("rows": [
+        agate.Row({"name": "index_a", "columns": ["column_a"], "type": "hash", "unique": False}),
+        agate.Row({"name": "index_b", "columns": ["time_dim_a"], "type": "btree", "unique": False}),
+    ])
+}
+"""
+RelationResults = Dict[str, Union[agate.Row, agate.Table]]
+
+
+@dataclass(frozen=True)
+class RelationConfigBase:
+    @classmethod
+    def from_dict(cls, kwargs_dict) -> "RelationConfigBase":
+        """
+        This assumes the subclass of `RelationConfigBase` is flat, in the sense that no attribute is
+        itself another subclass of `RelationConfigBase`. If that's not the case, this should be overriden
+        to manually manage that complexity.
+
+        Args:
+            kwargs_dict: the dict representation of this instance
+
+        Returns: the `RelationConfigBase` representation associated with the provided dict
+        """
+        return cls(**filter_null_values(kwargs_dict))  # type: ignore
+
+    @classmethod
+    def _not_implemented_error(cls) -> NotImplementedError:
+        return NotImplementedError(
+            "This relation type has not been fully configured for this adapter."
+        )

core/dbt/adapters/relation_configs/config_change.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Hashable
+
+from dbt.adapters.relation_configs.config_base import RelationConfigBase
+from dbt.dataclass_schema import StrEnum
+
+
+class RelationConfigChangeAction(StrEnum):
+    alter = "alter"
+    create = "create"
+    drop = "drop"
+
+
+@dataclass(frozen=True, eq=True, unsafe_hash=True)
+class RelationConfigChange(RelationConfigBase, ABC):
+    action: RelationConfigChangeAction
+    context: Hashable  # this is usually a RelationConfig, e.g. IndexConfig, but shouldn't be limited
+
+    @property
+    @abstractmethod
+    def requires_full_refresh(self) -> bool:
+        raise self._not_implemented_error()

core/dbt/adapters/relation_configs/config_validation.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+from typing import Set, Optional
+
+from dbt.exceptions import DbtRuntimeError
+
+
+@dataclass(frozen=True, eq=True, unsafe_hash=True)
+class RelationConfigValidationRule:
+    validation_check: bool
+    validation_error: Optional[DbtRuntimeError]
+
+    @property
+    def default_error(self):
+        return DbtRuntimeError(
+            "There was a validation error in preparing this relation config."
+            "No additional context was provided by this adapter."
+        )
+
+
+@dataclass(frozen=True)
+class RelationConfigValidationMixin:
+    def __post_init__(self):
+        self.run_validation_rules()
+
+    @property
+    def validation_rules(self) -> Set[RelationConfigValidationRule]:
+        """
+        A set of validation rules to run against the object upon creation.
+
+        A validation rule is a combination of a validation check (bool) and an optional error message.
+
+        This defaults to no validation rules if not implemented. It's recommended to override this with values,
+        but that may not always be necessary.
+
+        Returns: a set of validation rules
+        """
+        return set()
+
+    def run_validation_rules(self):
+        for validation_rule in self.validation_rules:
+            try:
+                assert validation_rule.validation_check
+            except AssertionError:
+                if validation_rule.validation_error:
+                    raise validation_rule.validation_error
+                else:
+                    raise validation_rule.default_error
+        self.run_child_validation_rules()
+
+    def run_child_validation_rules(self):
+        for attr_value in vars(self).values():
+            if hasattr(attr_value, "validation_rules"):
+                attr_value.run_validation_rules()
+            if isinstance(attr_value, set):
+                for member in attr_value:
+                    if hasattr(member, "validation_rules"):
+                        member.run_validation_rules()

core/dbt/context/exceptions_jinja.py

@@ -25,6 +25,7 @@
     RelationWrongTypeError,
     ContractError,
     ColumnTypeMissingError,
+    FailFastError,
 )
 
 
@@ -107,6 +108,10 @@ def column_type_missing(column_names) -> NoReturn:
     raise ColumnTypeMissingError(column_names)
 
 
+def raise_fail_fast_error(msg, node=None) -> NoReturn:
+    raise FailFastError(msg, node=node)
+
+
 # Update this when a new function should be added to the
 # dbt context's `exceptions` key!
 CONTEXT_EXPORTS = {
@@ -131,6 +136,7 @@ def column_type_missing(column_names) -> NoReturn:
         relation_wrong_type,
         raise_contract_error,
         column_type_missing,
+        raise_fail_fast_error,
     ]
 }
 

core/dbt/contracts/graph/model_config.py

@@ -2,15 +2,17 @@
 from enum import Enum
 from itertools import chain
 from typing import Any, List, Optional, Dict, Union, Type, TypeVar, Callable
+
 from dbt.dataclass_schema import (
     dbtClassMixin,
     ValidationError,
     register_pattern,
+    StrEnum,
 )
 from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed, Docs
 from dbt.contracts.graph.utils import validate_color
-from dbt.exceptions import DbtInternalError, CompilationError
 from dbt.contracts.util import Replaceable, list_str
+from dbt.exceptions import DbtInternalError, CompilationError
 from dbt import hooks
 from dbt.node_types import NodeType
 
@@ -189,6 +191,16 @@ class Severity(str):
 register_pattern(Severity, insensitive_patterns("warn", "error"))
 
 
+class OnConfigurationChangeOption(StrEnum):
+    Apply = "apply"
+    Continue = "continue"
+    Fail = "fail"
+
+    @classmethod
+    def default(cls) -> "OnConfigurationChangeOption":
+        return cls.Apply
+
+
 @dataclass
 class ContractConfig(dbtClassMixin, Replaceable):
     enforced: bool = False
@@ -287,11 +299,17 @@ def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> boo
                     return False
         return True
 
-    # This is used in 'add_config_call' to created the combined config_call_dict.
+    # This is used in 'add_config_call' to create the combined config_call_dict.
     # 'meta' moved here from node
     mergebehavior = {
         "append": ["pre-hook", "pre_hook", "post-hook", "post_hook", "tags"],
-        "update": ["quoting", "column_types", "meta", "docs", "contract"],
+        "update": [
+            "quoting",
+            "column_types",
+            "meta",
+            "docs",
+            "contract",
+        ],
         "dict_key_append": ["grants"],
     }
 
@@ -445,6 +463,9 @@ class NodeConfig(NodeAndTestConfig):
     # sometimes getting the Union order wrong, causing serialization failures.
     unique_key: Union[str, List[str], None] = None
     on_schema_change: Optional[str] = "ignore"
+    on_configuration_change: OnConfigurationChangeOption = field(
+        default_factory=OnConfigurationChangeOption.default
+    )
     grants: Dict[str, Any] = field(
         default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta()
     )

core/dbt/contracts/relation.py

@@ -17,7 +17,7 @@ class RelationType(StrEnum):
     Table = "table"
     View = "view"
     CTE = "cte"
-    MaterializedView = "materializedview"
+    MaterializedView = "materialized_view"
     External = "external"
 
 

core/dbt/include/global_project/macros/adapters/indexes.sql

@@ -21,3 +21,21 @@
     {% endif %}
   {% endfor %}
 {% endmacro %}
+
+
+{% macro get_drop_index_sql(relation, index_name) -%}
+    {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}
+{%- endmacro %}
+
+{% macro default__get_drop_index_sql(relation, index_name) -%}
+    {{ exceptions.raise_compiler_error("`get_drop_index_sql has not been implemented for this adapter.") }}
+{%- endmacro %}
+
+
+{% macro get_show_indexes_sql(relation) -%}
+    {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}
+{%- endmacro %}
+
+{% macro default__get_show_indexes_sql(relation) -%}
+    {{ exceptions.raise_compiler_error("`get_show_indexes_sql has not been implemented for this adapter.") }}
+{%- endmacro %}

core/dbt/include/global_project/macros/materializations/models/materialized_view/alter_materialized_view.sql

@@ -0,0 +1,30 @@
+{% macro get_alter_materialized_view_as_sql(
+    relation,
+    configuration_changes,
+    sql,
+    existing_relation,
+    backup_relation,
+    intermediate_relation
+) %}
+    {{- log('Applying ALTER to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(
+        relation,
+        configuration_changes,
+        sql,
+        existing_relation,
+        backup_relation,
+        intermediate_relation
+    ) -}}
+{% endmacro %}
+
+
+{% macro default__get_alter_materialized_view_as_sql(
+    relation,
+    configuration_changes,
+    sql,
+    existing_relation,
+    backup_relation,
+    intermediate_relation
+) %}
+    {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/models/materialized_view/create_materialized_view.sql

@@ -0,0 +1,9 @@
+{% macro get_create_materialized_view_as_sql(relation, sql) -%}
+    {{- log('Applying CREATE to: ' ~ relation) -}}
+    {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}
+{%- endmacro %}
+
+
+{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}
+    {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/models/materialized_view/get_materialized_view_configuration_changes.sql

@@ -0,0 +1,23 @@
+{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}
+    /* {#
+    It's recommended that configuration changes be formatted as follows:
+    {"<change_category>": [{"action": "<name>", "context": ...}]}
+
+    For example:
+    {
+        "indexes": [
+            {"action": "drop", "context": "index_abc"},
+            {"action": "create", "context": {"columns": ["column_1", "column_2"], "type": "hash", "unique": True}},
+        ],
+    }
+
+    Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.
+    #} */
+    {{- log('Determining configuration changes on: ' ~ existing_relation) -}}
+    {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}
+{% endmacro %}
+
+
+{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}
+    {{ exceptions.raise_compiler_error("Materialized views have not been implemented for this adapter.") }}
+{% endmacro %}