From 9718e4f37ea2b14b64477aad5bccda198cfb87e7 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:12:55 -0800 Subject: [PATCH 1/8] add unique UUID to temp table name --- dbt/adapters/duckdb/impl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index b24c01db..996cf869 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -1,9 +1,11 @@ +from collections import defaultdict import os from typing import Any from typing import List from typing import Optional from typing import Sequence from typing import TYPE_CHECKING +from uuid import uuid4 from dbt_common.contracts.constraints import ColumnLevelConstraint from dbt_common.contracts.constraints import ConstraintType @@ -50,6 +52,7 @@ class DuckDBAdapter(SQLAdapter): # can be overridden via the model config metadata _temp_schema_name = DEFAULT_TEMP_SCHEMA_NAME + _temp_schema_unique_id = defaultdict(uuid4) @classmethod def date_function(cls) -> str: @@ -286,10 +289,12 @@ def get_temp_relation_path(self, model: Any): currently doesn't support remote temporary tables. Instead we use a regular table that is dropped at the end of the incremental macro or post-model hook. """ + # Add a unique identifier for this Python session to enable running the incremental model concurrently + unique_id = self._temp_schema_unique_id[model.identifier] return Path( schema=self._temp_schema_name, database=model.database, - identifier=model.identifier, + identifier=f"{model.identifier}__{unique_id}", ) def post_model_hook(self, config: Any, context: Any) -> None: From 5b53e0f8bb1639823924e72b0ec8a87a75486eee Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:17:34 -0800 Subject: [PATCH 2/8] rename to uuid --- dbt/adapters/duckdb/impl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 996cf869..d682440c 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -52,7 +52,7 @@ class DuckDBAdapter(SQLAdapter): # can be overridden via the model config metadata _temp_schema_name = DEFAULT_TEMP_SCHEMA_NAME - _temp_schema_unique_id = defaultdict(uuid4) + _temp_schema_model_uuid = defaultdict(uuid4) @classmethod def date_function(cls) -> str: @@ -289,12 +289,12 @@ def get_temp_relation_path(self, model: Any): currently doesn't support remote temporary tables. Instead we use a regular table that is dropped at the end of the incremental macro or post-model hook. """ - # Add a unique identifier for this Python session to enable running the incremental model concurrently - unique_id = self._temp_schema_unique_id[model.identifier] + # Add a unique identifier for this model (scoped per dbt run) + uuid = self._temp_schema_model_uuid[model.identifier] return Path( schema=self._temp_schema_name, database=model.database, - identifier=f"{model.identifier}__{unique_id}", + identifier=f"{model.identifier}__{uuid}", ) def post_model_hook(self, config: Any, context: Any) -> None: From a6f0480a64600a7e39e6ec523a82360ac8e5c3ce Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:19:03 -0800 Subject: [PATCH 3/8] make private variable --- dbt/adapters/duckdb/impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index d682440c..4b257b9e 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -290,11 +290,11 @@ def get_temp_relation_path(self, model: Any): table that is dropped at the end of the incremental macro or post-model hook. """ # Add a unique identifier for this model (scoped per dbt run) - uuid = self._temp_schema_model_uuid[model.identifier] + _uuid = self._temp_schema_model_uuid[model.identifier] return Path( schema=self._temp_schema_name, database=model.database, - identifier=f"{model.identifier}__{uuid}", + identifier=f"{model.identifier}__{_uuid}", ) def post_model_hook(self, config: Any, context: Any) -> None: From ca5846cc7b1fa21061df5f60c516483c45fbd1cf Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:23:52 -0800 Subject: [PATCH 4/8] Add type annotation --- dbt/adapters/duckdb/impl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 4b257b9e..24f47ee6 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -5,7 +5,7 @@ from typing import Optional from typing import Sequence from typing import TYPE_CHECKING -from uuid import uuid4 +from uuid import UUID, uuid4 from dbt_common.contracts.constraints import ColumnLevelConstraint from dbt_common.contracts.constraints import ConstraintType @@ -52,7 +52,7 @@ class DuckDBAdapter(SQLAdapter): # can be overridden via the model config metadata _temp_schema_name = DEFAULT_TEMP_SCHEMA_NAME - _temp_schema_model_uuid = defaultdict(uuid4) + _temp_schema_model_uuid: dict[str, UUID] = defaultdict(uuid4) @classmethod def date_function(cls) -> str: From c92e2bd3a3665cbe6459cc21e4bfe9a76696977d Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:27:10 -0800 Subject: [PATCH 5/8] formatting --- dbt/adapters/duckdb/impl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 24f47ee6..06f9dc2e 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -1,11 +1,12 @@ -from collections import defaultdict import os +from collections import defaultdict from typing import Any from typing import List from typing import Optional from typing import Sequence from typing import TYPE_CHECKING -from uuid import UUID, uuid4 +from uuid import UUID +from uuid import uuid4 from dbt_common.contracts.constraints import ColumnLevelConstraint from dbt_common.contracts.constraints import ConstraintType From fb8e0922cbe4a4c02974b107edce28951455dfa3 Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:44:44 -0800 Subject: [PATCH 6/8] remove dash, use only last part of uuid --- dbt/adapters/duckdb/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 06f9dc2e..3d0e364f 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -53,7 +53,7 @@ class DuckDBAdapter(SQLAdapter): # can be overridden via the model config metadata _temp_schema_name = DEFAULT_TEMP_SCHEMA_NAME - _temp_schema_model_uuid: dict[str, UUID] = defaultdict(uuid4) + _temp_schema_model_uuid: dict[str, UUID] = defaultdict(lambda: str(uuid4()).split("-")[-1]) @classmethod def date_function(cls) -> str: From 87b1a57b9d5d50cf4881410765329a638d83751d Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:46:52 -0800 Subject: [PATCH 7/8] fix type hint --- dbt/adapters/duckdb/impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index 3d0e364f..c7d44be1 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -53,7 +53,7 @@ class DuckDBAdapter(SQLAdapter): # can be overridden via the model config metadata _temp_schema_name = DEFAULT_TEMP_SCHEMA_NAME - _temp_schema_model_uuid: dict[str, UUID] = defaultdict(lambda: str(uuid4()).split("-")[-1]) + _temp_schema_model_uuid: dict[str, str] = defaultdict(lambda: str(uuid4()).split("-")[-1]) @classmethod def date_function(cls) -> str: From 75894d34436b7893688499db88e36c4967da278f Mon Sep 17 00:00:00 2001 From: Guen Prawiroatmodjo Date: Tue, 3 Dec 2024 15:49:53 -0800 Subject: [PATCH 8/8] remove unused import --- dbt/adapters/duckdb/impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt/adapters/duckdb/impl.py b/dbt/adapters/duckdb/impl.py index c7d44be1..b62b5073 100644 --- a/dbt/adapters/duckdb/impl.py +++ b/dbt/adapters/duckdb/impl.py @@ -5,7 +5,6 @@ from typing import Optional from typing import Sequence from typing import TYPE_CHECKING -from uuid import UUID from uuid import uuid4 from dbt_common.contracts.constraints import ColumnLevelConstraint