From e7c15a2e8d8bfb98d9d311da071b0fe598d83234 Mon Sep 17 00:00:00 2001 From: Anders Swanson Date: Wed, 11 Oct 2023 11:24:05 -0400 Subject: [PATCH] drop bespoke mat, intro simpler macros --- .../create_materialized_view_as.sql | 9 ---- .../materialized_view/materialized_view.sql | 43 ---------------- .../refresh_materialized_view.sql | 7 --- .../relations/materialized_view/alter.sql | 50 +++++++++++++++++++ .../relations/materialized_view/create.sql | 8 +++ .../relations/materialized_view/describe.sql | 5 ++ .../relations/materialized_view/drop.sql | 3 ++ .../relations/materialized_view/refresh.sql | 3 ++ .../relations/materialized_view/rename.sql | 3 ++ 9 files changed, 72 insertions(+), 59 deletions(-) delete mode 100644 dbt/include/databricks/macros/materializations/materialized_view/create_materialized_view_as.sql delete mode 100644 dbt/include/databricks/macros/materializations/materialized_view/materialized_view.sql delete mode 100644 dbt/include/databricks/macros/materializations/materialized_view/refresh_materialized_view.sql create mode 100644 dbt/include/databricks/macros/relations/materialized_view/alter.sql create mode 100644 dbt/include/databricks/macros/relations/materialized_view/create.sql create mode 100644 dbt/include/databricks/macros/relations/materialized_view/describe.sql create mode 100644 dbt/include/databricks/macros/relations/materialized_view/drop.sql create mode 100644 dbt/include/databricks/macros/relations/materialized_view/refresh.sql create mode 100644 dbt/include/databricks/macros/relations/materialized_view/rename.sql diff --git a/dbt/include/databricks/macros/materializations/materialized_view/create_materialized_view_as.sql b/dbt/include/databricks/macros/materializations/materialized_view/create_materialized_view_as.sql deleted file mode 100644 index 85801bb9c..000000000 --- a/dbt/include/databricks/macros/materializations/materialized_view/create_materialized_view_as.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro get_create_materialized_view_as_sql(relation, sql) -%} - {{ adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) }} -{%- endmacro %} - -{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%} - create materialized view {{ relation }} - as - {{ sql }} -{% endmacro %} \ No newline at end of file diff --git a/dbt/include/databricks/macros/materializations/materialized_view/materialized_view.sql b/dbt/include/databricks/macros/materializations/materialized_view/materialized_view.sql deleted file mode 100644 index bf278f101..000000000 --- a/dbt/include/databricks/macros/materializations/materialized_view/materialized_view.sql +++ /dev/null @@ -1,43 +0,0 @@ -{% materialization materialized_view, adapter='databricks' %} - - {%- set identifier = model['alias'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set exists_as_materialized_view = (old_relation is not none and old_relation.is_materialized_view) -%} - - {%- set target_relation = api.Relation.create( - identifier=identifier, schema=schema, database=database, - type='materialized_view') -%} - {% set grant_config = config.get('grants') %} - - {{ run_hooks(pre_hooks) }} - - {%- set full_refresh_mode = should_full_refresh() -%} - - {%- if exists_as_materialized_view and not full_refresh_mode -%} - -- refresh materialized view - {% call statement('main') -%} - {{ get_refresh_materialized_view_sql(target_relation) }} - {%- endcall %} - {%- else -%} - -- If there's a table with the same name and we weren't told to full refresh, - -- that's an error. If we were told to full refresh, drop it. This behavior differs - -- for Snowflake and BigQuery, so multiple dispatch is used. - {%- if old_relation is not none and (not exists_as_materialized_view or full_refresh_mode) -%} - {{ handle_existing_table(full_refresh_mode, old_relation) }} - {%- endif -%} - - -- build model - {% call statement('main') -%} - {{ get_create_materialized_view_as_sql(target_relation, sql) }} - {%- endcall %} - {%- endif -%} - - {% set should_revoke = should_revoke(exists_as_materialized_view, full_refresh_mode) %} - {% do apply_grants(target_relation, grant_config, should_revoke) %} - - {{ run_hooks(post_hooks) }} - - {{ return({'relations': [target_relation]}) }} - -{% endmaterialization %} \ No newline at end of file diff --git a/dbt/include/databricks/macros/materializations/materialized_view/refresh_materialized_view.sql b/dbt/include/databricks/macros/materializations/materialized_view/refresh_materialized_view.sql deleted file mode 100644 index 47a99d243..000000000 --- a/dbt/include/databricks/macros/materializations/materialized_view/refresh_materialized_view.sql +++ /dev/null @@ -1,7 +0,0 @@ -{% macro get_refresh_materialized_view_sql(relation) -%} - {{ adapter.dispatch('get_refresh_materialized_view_sql', 'dbt')(relation) }} -{%- endmacro %} - -{% macro databricks__get_refresh_materialized_view_sql(relation) -%} - refresh materialized view {{ relation }} -{% endmacro %} \ No newline at end of file diff --git a/dbt/include/databricks/macros/relations/materialized_view/alter.sql b/dbt/include/databricks/macros/relations/materialized_view/alter.sql new file mode 100644 index 000000000..45c9b30eb --- /dev/null +++ b/dbt/include/databricks/macros/relations/materialized_view/alter.sql @@ -0,0 +1,50 @@ +{% macro databricks__get_alter_materialized_view_as_sql( + relation, + configuration_changes, + sql, + existing_relation, + backup_relation, + intermediate_relation +) %} + + -- apply a full refresh immediately if needed + {% if configuration_changes.requires_full_refresh %} + + {{ get_replace_sql(existing_relation, relation, sql) }} + + -- otherwise apply individual changes as needed + {% else %} + + {{ databricks__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }} + + {%- endif -%} + +{% endmacro %} + + +{%- macro databricks__update_indexes_on_materialized_view(relation, index_changes) -%} + {{- log("Applying UPDATE INDEXES to: " ~ relation) -}} + + {%- for _index_change in index_changes -%} + {%- set _index = _index_change.context -%} + + {%- if _index_change.action == "drop" -%} + + {{ databricks__get_drop_index_sql(relation, _index.name) }}; + + {%- elif _index_change.action == "create" -%} + + {{ databricks__get_create_index_sql(relation, _index.as_node_config) }} + + {%- endif -%} + + {%- endfor -%} + +{%- endmacro -%} + + +{% macro databricks__get_materialized_view_configuration_changes(existing_relation, new_config) %} + {% set _existing_materialized_view = databricks__describe_materialized_view(existing_relation) %} + {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %} + {% do return(_configuration_changes) %} +{% endmacro %} diff --git a/dbt/include/databricks/macros/relations/materialized_view/create.sql b/dbt/include/databricks/macros/relations/materialized_view/create.sql new file mode 100644 index 000000000..7aa5af68b --- /dev/null +++ b/dbt/include/databricks/macros/relations/materialized_view/create.sql @@ -0,0 +1,8 @@ +{% macro databricks__get_create_materialized_view_as_sql(relation, sql) %} + create materialized view if not exists {{ relation }} as {{ sql }}; + + {% for _index_dict in config.get('indexes', []) -%} + {{- get_create_index_sql(relation, _index_dict) -}} + {%- endfor -%} + +{% endmacro %} diff --git a/dbt/include/databricks/macros/relations/materialized_view/describe.sql b/dbt/include/databricks/macros/relations/materialized_view/describe.sql new file mode 100644 index 000000000..3d701d04e --- /dev/null +++ b/dbt/include/databricks/macros/relations/materialized_view/describe.sql @@ -0,0 +1,5 @@ +{% macro databricks__describe_materialized_view(relation) %} + -- for now just get the indexes, we don't need the name or the query yet + {% set _indexes = run_query(get_show_indexes_sql(relation)) %} + {% do return({'indexes': _indexes}) %} +{% endmacro %} diff --git a/dbt/include/databricks/macros/relations/materialized_view/drop.sql b/dbt/include/databricks/macros/relations/materialized_view/drop.sql new file mode 100644 index 000000000..19e5fc29e --- /dev/null +++ b/dbt/include/databricks/macros/relations/materialized_view/drop.sql @@ -0,0 +1,3 @@ +{% macro databricks__drop_materialized_view(relation) -%} + drop materialized view if exists {{ relation }} cascade +{%- endmacro %} diff --git a/dbt/include/databricks/macros/relations/materialized_view/refresh.sql b/dbt/include/databricks/macros/relations/materialized_view/refresh.sql new file mode 100644 index 000000000..3738970e4 --- /dev/null +++ b/dbt/include/databricks/macros/relations/materialized_view/refresh.sql @@ -0,0 +1,3 @@ +{% macro databricks__refresh_materialized_view(relation) %} + refresh materialized view {{ relation }} +{% endmacro %} diff --git a/dbt/include/databricks/macros/relations/materialized_view/rename.sql b/dbt/include/databricks/macros/relations/materialized_view/rename.sql new file mode 100644 index 000000000..9e5287bea --- /dev/null +++ b/dbt/include/databricks/macros/relations/materialized_view/rename.sql @@ -0,0 +1,3 @@ +{% macro databricks__get_rename_materialized_view_sql(relation, new_name) %} + alter materialized view {{ relation }} rename to {{ new_name }} +{% endmacro %}