Skip to content

Commit

Permalink
feat(ingest/dbt): add support for latest dbt version 1.3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@MatthieuBlais
Matthieu Blais authored and MatthieuBlais committed Dec 6, 2022
1 parent 7065971 commit 87d3996
Show file tree
Hide file tree
Showing 14 changed files with 2,352 additions and 867 deletions.
23 changes: 15 additions & 8 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,8 @@ class DBTNode:
alias: Optional[str] # alias if present
comment: str
description: str
raw_sql: Optional[str]
language: str
raw_code: Optional[str]

dbt_adapter: str
dbt_name: str
Expand All @@ -375,7 +376,7 @@ class DBTNode:
query_tag: Dict[str, Any] = field(default_factory=dict)

tags: List[str] = field(default_factory=list)
compiled_sql: Optional[str] = None
compiled_code: Optional[str] = None

test_info: Optional["DBTTest"] = None # only populated if node_type == 'test'
test_result: Optional["DBTTestResult"] = None
Expand Down Expand Up @@ -410,7 +411,13 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
custom_properties = node.meta

# additional node attributes to extract to custom properties
node_attributes = ["node_type", "materialization", "dbt_file_path", "catalog_type"]
node_attributes = [
"node_type",
"materialization",
"dbt_file_path",
"catalog_type",
"language",
]

for attribute in node_attributes:
node_attribute_value = getattr(node, attribute)
Expand Down Expand Up @@ -834,7 +841,7 @@ def _make_assertion_from_test(
mce_builder.make_schema_field_urn(upstream_urn, column_name)
],
nativeType=node.name,
logic=node.compiled_sql if node.compiled_sql else node.raw_sql,
logic=node.compiled_code if node.compiled_code else node.raw_code,
aggregation=AssertionStdAggregationClass._NATIVE_,
nativeParameters=string_map(kw_args),
),
Expand All @@ -848,7 +855,7 @@ def _make_assertion_from_test(
dataset=upstream_urn,
scope=DatasetAssertionScopeClass.DATASET_ROWS,
operator=AssertionStdOperatorClass._NATIVE_,
logic=node.compiled_sql if node.compiled_sql else node.raw_sql,
logic=node.compiled_code if node.compiled_code else node.raw_code,
nativeType=node.name,
aggregation=AssertionStdAggregationClass._NATIVE_,
nativeParameters=string_map(kw_args),
Expand Down Expand Up @@ -1023,7 +1030,7 @@ def create_platform_mces(
aspects.append(upstream_lineage_class)

# add view properties aspect
if node.raw_sql:
if node.raw_code and node.language == "sql":
view_prop_aspect = self._create_view_properties_aspect(node)
aspects.append(view_prop_aspect)

Expand Down Expand Up @@ -1157,11 +1164,11 @@ def get_external_url(self, node: DBTNode) -> Optional[str]:
def _create_view_properties_aspect(self, node: DBTNode) -> ViewPropertiesClass:
materialized = node.materialization in {"table", "incremental"}
# this function is only called when raw sql is present. assert is added to satisfy lint checks
assert node.raw_sql is not None
assert node.raw_code is not None
view_properties = ViewPropertiesClass(
materialized=materialized,
viewLanguage="SQL",
viewLogic=node.raw_sql,
viewLogic=node.raw_code,
)
return view_properties

Expand Down
11 changes: 9 additions & 2 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,15 +222,22 @@ def extract_dbt_entities(
max_loaded_at=max_loaded_at,
comment=comment,
description=manifest_node.get("description", ""),
raw_sql=manifest_node.get("raw_sql"),
raw_code=manifest_node.get(
"raw_code", manifest_node.get("raw_sql")
), # Backward compatibility dbt <=v1.2
language=manifest_node.get(
"language", "sql"
), # Backward compatibility dbt <=v1.2
upstream_nodes=upstream_nodes,
materialization=materialization,
catalog_type=catalog_type,
meta=meta,
query_tag=query_tag_props,
tags=tags,
owner=owner,
compiled_sql=manifest_node.get("compiled_sql"),
compiled_code=manifest_node.get(
"compiled_code", manifest_node.get("compiled_sql")
), # Backward compatibility dbt <=v1.2
test_info=test_info,
)

Expand Down
15 changes: 15 additions & 0 deletions metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -154,6 +155,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -263,6 +265,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -413,6 +416,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -618,6 +622,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -800,6 +805,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -922,6 +928,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1059,6 +1066,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1195,6 +1203,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1401,6 +1410,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1563,6 +1573,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1735,6 +1746,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1893,6 +1905,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2051,6 +2064,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2209,6 +2223,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
Expand Down
16 changes: 16 additions & 0 deletions metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -204,6 +205,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -371,6 +373,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -585,6 +588,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -717,6 +721,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -878,6 +883,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1060,6 +1066,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1182,6 +1189,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1319,6 +1327,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1458,6 +1467,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1664,6 +1674,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1826,6 +1837,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2001,6 +2013,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2159,6 +2172,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2317,6 +2331,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2475,6 +2490,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down
Loading

0 comments on commit 87d3996

Please sign in to comment.