From 13e5655477610f9aa002aa4b8a0c7cc4af6a9cfe Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 27 Aug 2024 21:54:32 -0400 Subject: [PATCH 1/7] Update snapshot_merge.sql --- .../postgres/macros/materializations/snapshot_merge.sql | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dbt/include/postgres/macros/materializations/snapshot_merge.sql b/dbt/include/postgres/macros/materializations/snapshot_merge.sql index 807c70b6..6db5d458 100644 --- a/dbt/include/postgres/macros/materializations/snapshot_merge.sql +++ b/dbt/include/postgres/macros/materializations/snapshot_merge.sql @@ -2,12 +2,15 @@ {% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} + {%- set dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" -%} + {%- set dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" -%} + update {{ target }} - set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to + set {{ dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ dbt_valid_to }} from {{ source }} as DBT_INTERNAL_SOURCE - where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text + where DBT_INTERNAL_SOURCE.{{ dbt_scd_id }}::text = {{ target }}.{{ dbt_scd_id }}::text and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text) - and {{ target }}.dbt_valid_to is null; + and {{ target }}.{{ dbt_valid_to }} is null; insert into {{ target }} ({{ insert_cols_csv }}) select {% for column in insert_cols -%} From a40cfeccf71e44acbc9a82e721a293a0b10f71f9 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 3 Sep 2024 16:04:50 -0400 Subject: [PATCH 2/7] Changie and update branches in pyproject.toml --- .changes/unreleased/Features-20240903-160221.yaml | 6 ++++++ pyproject.toml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Features-20240903-160221.yaml diff --git a/.changes/unreleased/Features-20240903-160221.yaml b/.changes/unreleased/Features-20240903-160221.yaml new file mode 100644 index 00000000..8409e248 --- /dev/null +++ b/.changes/unreleased/Features-20240903-160221.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow configuring snapshot column names +time: 2024-09-03T16:02:21.069085-04:00 +custom: + Author: gshank + Issue: "144" diff --git a/pyproject.toml b/pyproject.toml index e6848a4f..0246950e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,10 +54,10 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@snapshot_column_names", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@snapshot_column_names#subdirectory=core", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', "freezegun", From 39525bacc4bd1494f9e0e28eedcf6d14e699354d Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 12 Sep 2024 17:31:51 -0400 Subject: [PATCH 3/7] Update to use column names --- .../postgres/macros/materializations/snapshot_merge.sql | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbt/include/postgres/macros/materializations/snapshot_merge.sql b/dbt/include/postgres/macros/materializations/snapshot_merge.sql index 6db5d458..4c9cf9fe 100644 --- a/dbt/include/postgres/macros/materializations/snapshot_merge.sql +++ b/dbt/include/postgres/macros/materializations/snapshot_merge.sql @@ -2,15 +2,14 @@ {% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - {%- set dbt_scd_id = config.get("dbt_scd_id_column_name") or "dbt_scd_id" -%} - {%- set dbt_valid_to = config.get("dbt_valid_to_column_name") or "dbt_valid_to" -%} + {%- set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} update {{ target }} - set {{ dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ dbt_valid_to }} + set {{ stcn.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ stcn.dbt_valid_to }} from {{ source }} as DBT_INTERNAL_SOURCE - where DBT_INTERNAL_SOURCE.{{ dbt_scd_id }}::text = {{ target }}.{{ dbt_scd_id }}::text + where DBT_INTERNAL_SOURCE.{{ stcn.dbt_scd_id }}::text = {{ target }}.{{ stcn.dbt_scd_id }}::text and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text) - and {{ target }}.{{ dbt_valid_to }} is null; + and {{ target }}.{{ stcn.dbt_valid_to }} is null; insert into {{ target }} ({{ insert_cols_csv }}) select {% for column in insert_cols -%} From 24774877514526f0477fa837de115fd74f5aaa49 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 13 Sep 2024 17:12:49 -0400 Subject: [PATCH 4/7] remove branch references --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0246950e..e6848a4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,10 +54,10 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@snapshot_column_names", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git@snapshot_column_names#subdirectory=core", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', "freezegun", From 1914c7ce0565b03e641f528f03daf1de936b0247 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 17 Sep 2024 11:59:49 -0400 Subject: [PATCH 5/7] Update dev-requirements.txt --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e6848a4f..3aa7d5df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@snapshot_column_names", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", From b8a7d1f49d650b5e71f39bb3155505a14d664c0f Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 18 Sep 2024 13:43:18 -0400 Subject: [PATCH 6/7] Change stcn to columns --- .../postgres/macros/materializations/snapshot_merge.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/include/postgres/macros/materializations/snapshot_merge.sql b/dbt/include/postgres/macros/materializations/snapshot_merge.sql index 4c9cf9fe..894ea4b4 100644 --- a/dbt/include/postgres/macros/materializations/snapshot_merge.sql +++ b/dbt/include/postgres/macros/materializations/snapshot_merge.sql @@ -2,14 +2,14 @@ {% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} - {%- set stcn = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} + {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%} update {{ target }} - set {{ stcn.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ stcn.dbt_valid_to }} + set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} from {{ source }} as DBT_INTERNAL_SOURCE - where DBT_INTERNAL_SOURCE.{{ stcn.dbt_scd_id }}::text = {{ target }}.{{ stcn.dbt_scd_id }}::text + where DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }}::text = {{ target }}.{{ columns.dbt_scd_id }}::text and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text) - and {{ target }}.{{ stcn.dbt_valid_to }} is null; + and {{ target }}.{{ columns.dbt_valid_to }} is null; insert into {{ target }} ({{ insert_cols_csv }}) select {% for column in insert_cols -%} From c57c61f3cb5328e90ddebd0c6024949072729057 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 19 Sep 2024 09:23:58 -0400 Subject: [PATCH 7/7] Remove branch name and update dbt-adapters pin to >= 1.7.0 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3aa7d5df..726dc64a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ "psycopg2-binary>=2.9,<3.0", - "dbt-adapters>=1.1.1,<2.0", + "dbt-adapters>=1.7.0,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", # installed via dbt-adapters but used directly @@ -54,7 +54,7 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@snapshot_column_names", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",