Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ingest/dbt): add support for latest DBT version 1.3 #6651

Merged
merged 4 commits into from
Dec 7, 2022
+1,941 −400
Merged

feat(ingest/dbt): add support for latest DBT version 1.3 #6651

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
87d3996
feat(ingest/dbt): add support for latest dbt version 1.3.0
Dec 6, 2022
a70978e
feat(ingest/dbt): reduce golden state noise in mr
Dec 6, 2022
e95e41d
update for dbt cloud
hsheth2 Dec 6, 2022
cd17d03
another fix for dbt cloud
hsheth2 Dec 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,11 +278,11 @@ def _parse_into_dbt_node(self, node: Dict) -> DBTNode:
)

# The code fields are new in dbt 1.3, and replace the sql ones.
raw_sql = node["rawCode"] or node["rawSql"]
compiled_sql = node["compiledCode"] or node["compiledSql"]
raw_code = node["rawCode"] or node["rawSql"]
compiled_code = node["compiledCode"] or node["compiledSql"]
else:
raw_sql = None
compiled_sql = None
raw_code = None
compiled_code = None

max_loaded_at = None
if node["resourceType"] == "source":
Expand Down Expand Up @@ -369,8 +369,9 @@ def _parse_into_dbt_node(self, node: Dict) -> DBTNode:
query_tag={}, # TODO: Get this from the dbt API.
tags=tags,
owner=owner,
raw_sql=raw_sql,
compiled_sql=compiled_sql,
language="sql", # TODO: dbt Cloud doesn't surface this
raw_code=raw_code,
compiled_code=compiled_code,
columns=columns,
test_info=test_info,
test_result=test_result,
Expand Down
23 changes: 15 additions & 8 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,8 @@ class DBTNode:
alias: Optional[str] # alias if present
comment: str
description: str
raw_sql: Optional[str]
language: Optional[str]
raw_code: Optional[str]

dbt_adapter: str
dbt_name: str
Expand All @@ -375,7 +376,7 @@ class DBTNode:
query_tag: Dict[str, Any] = field(default_factory=dict)

tags: List[str] = field(default_factory=list)
compiled_sql: Optional[str] = None
compiled_code: Optional[str] = None

test_info: Optional["DBTTest"] = None # only populated if node_type == 'test'
test_result: Optional["DBTTestResult"] = None
Expand Down Expand Up @@ -410,7 +411,13 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
custom_properties = node.meta

# additional node attributes to extract to custom properties
node_attributes = ["node_type", "materialization", "dbt_file_path", "catalog_type"]
node_attributes = [
"node_type",
"materialization",
"dbt_file_path",
"catalog_type",
"language",
]

for attribute in node_attributes:
node_attribute_value = getattr(node, attribute)
Expand Down Expand Up @@ -834,7 +841,7 @@ def _make_assertion_from_test(
mce_builder.make_schema_field_urn(upstream_urn, column_name)
],
nativeType=node.name,
logic=node.compiled_sql if node.compiled_sql else node.raw_sql,
logic=node.compiled_code if node.compiled_code else node.raw_code,
aggregation=AssertionStdAggregationClass._NATIVE_,
nativeParameters=string_map(kw_args),
),
Expand All @@ -848,7 +855,7 @@ def _make_assertion_from_test(
dataset=upstream_urn,
scope=DatasetAssertionScopeClass.DATASET_ROWS,
operator=AssertionStdOperatorClass._NATIVE_,
logic=node.compiled_sql if node.compiled_sql else node.raw_sql,
logic=node.compiled_code if node.compiled_code else node.raw_code,
nativeType=node.name,
aggregation=AssertionStdAggregationClass._NATIVE_,
nativeParameters=string_map(kw_args),
Expand Down Expand Up @@ -1023,7 +1030,7 @@ def create_platform_mces(
aspects.append(upstream_lineage_class)

# add view properties aspect
if node.raw_sql:
if node.raw_code and node.language == "sql":
view_prop_aspect = self._create_view_properties_aspect(node)
aspects.append(view_prop_aspect)

Expand Down Expand Up @@ -1157,11 +1164,11 @@ def get_external_url(self, node: DBTNode) -> Optional[str]:
def _create_view_properties_aspect(self, node: DBTNode) -> ViewPropertiesClass:
materialized = node.materialization in {"table", "incremental"}
# this function is only called when raw sql is present. assert is added to satisfy lint checks
assert node.raw_sql is not None
assert node.raw_code is not None
view_properties = ViewPropertiesClass(
materialized=materialized,
viewLanguage="SQL",
viewLogic=node.raw_sql,
viewLogic=node.raw_code,
)
return view_properties

Expand Down
11 changes: 9 additions & 2 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,15 +222,22 @@ def extract_dbt_entities(
max_loaded_at=max_loaded_at,
comment=comment,
description=manifest_node.get("description", ""),
raw_sql=manifest_node.get("raw_sql"),
raw_code=manifest_node.get(
"raw_code", manifest_node.get("raw_sql")
), # Backward compatibility dbt <=v1.2
language=manifest_node.get(
"language", "sql"
), # Backward compatibility dbt <=v1.2
upstream_nodes=upstream_nodes,
materialization=materialization,
catalog_type=catalog_type,
meta=meta,
query_tag=query_tag_props,
tags=tags,
owner=owner,
compiled_sql=manifest_node.get("compiled_sql"),
compiled_code=manifest_node.get(
"compiled_code", manifest_node.get("compiled_sql")
), # Backward compatibility dbt <=v1.2
test_info=test_info,
)

Expand Down
15 changes: 15 additions & 0 deletions metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -154,6 +155,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -263,6 +265,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -413,6 +416,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -618,6 +622,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -800,6 +805,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -922,6 +928,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1059,6 +1066,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1195,6 +1203,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1401,6 +1410,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1563,6 +1573,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1735,6 +1746,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1893,6 +1905,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2051,6 +2064,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2209,6 +2223,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down
16 changes: 16 additions & 0 deletions metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -204,6 +205,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -371,6 +373,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -585,6 +588,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -717,6 +721,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -878,6 +883,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1060,6 +1066,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1182,6 +1189,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1319,6 +1327,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1458,6 +1467,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1664,6 +1674,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1826,6 +1837,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2001,6 +2013,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2159,6 +2172,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2317,6 +2331,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2475,6 +2490,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down
Loading