Skip to content

Commit

Permalink
feat(ingest/dbt): include package_name in dbt custom props (datahub-p…
Browse files Browse the repository at this point in the history 
…roject#10652)
  • Loading branch information
@hsheth2 @sleeperdeep
hsheth2 authored and sleeperdeep committed Jun 25, 2024
1 parent d28aec3 commit e318ce5
Show file tree
Hide file tree
Showing 11 changed files with 126 additions and 0 deletions.
2 changes: 2 additions & 0 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ def set_metadata_endpoint(cls, values: dict) -> dict:
"""

_DBT_GRAPHQL_MODEL_SEED_SNAPSHOT_FIELDS = """
packageName
alias
error
status
Expand Down Expand Up @@ -433,6 +434,7 @@ def _parse_into_dbt_node(self, node: Dict) -> DBTNode:
dbt_name=key,
# TODO: Get the dbt adapter natively.
dbt_adapter=self.config.target_platform,
dbt_package_name=node.get("packageName"),
database=node.get("database"),
schema=node.get("schema"),
name=name,
Expand Down
2 changes: 2 additions & 0 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,6 +502,7 @@ class DBTNode:
dbt_adapter: str
dbt_name: str
dbt_file_path: Optional[str]
dbt_package_name: Optional[str] # this is pretty much always present

node_type: str # source, model, snapshot, seed, test, etc
max_loaded_at: Optional[datetime]
Expand Down Expand Up @@ -644,6 +645,7 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
"catalog_type": node.catalog_type,
"language": node.language,
"dbt_unique_id": node.dbt_name,
"dbt_package_name": node.dbt_package_name,
}

for attribute, node_attribute_value in node_attributes.items():
Expand Down
1 change: 1 addition & 0 deletions metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ def extract_dbt_entities(
dbtNode = DBTNode(
dbt_name=key,
dbt_adapter=manifest_adapter,
dbt_package_name=manifest_node.get("package_name"),
database=manifest_node["database"],
schema=manifest_node["schema"],
name=name,
Expand Down
16 changes: 16 additions & 0 deletions metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -416,6 +417,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -643,6 +645,7 @@
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -979,6 +982,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1170,6 +1174,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1401,6 +1406,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1696,6 +1702,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1876,6 +1883,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2082,6 +2090,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2280,6 +2289,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2621,6 +2631,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2874,6 +2885,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3141,6 +3153,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3390,6 +3403,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3639,6 +3653,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3888,6 +3903,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
Expand Down
17 changes: 17 additions & 0 deletions metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -376,6 +377,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -596,6 +598,7 @@
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -924,6 +927,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1141,6 +1145,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "snapshot.sample_dbt.customer_snapshot",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1541,6 +1546,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -1766,6 +1772,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2061,6 +2068,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2241,6 +2249,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2447,6 +2456,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2645,6 +2655,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -2986,6 +2997,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3239,6 +3251,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3506,6 +3519,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -3755,6 +3769,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -4004,6 +4019,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down Expand Up @@ -4253,6 +4269,7 @@
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"dbt_package_name": "sample_dbt",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
Expand Down
Loading

0 comments on commit e318ce5

Please sign in to comment.