From b4820f9e5fb6b30205019547aa3f571fe2539a97 Mon Sep 17 00:00:00 2001 From: MugdhaHardikar-GSLab Date: Fri, 15 Jul 2022 22:46:57 +0530 Subject: [PATCH 01/10] fix(dbt): fix issue of dbt stateful ingestion with tests --- .../src/datahub/emitter/mce_builder.py | 11 +++++++++-- .../src/datahub/ingestion/source/dbt.py | 15 +++++++++++---- .../ingestion/source/state/sql_common_state.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index 5936546090acbc..5be422e9dc400a 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -15,6 +15,7 @@ from datahub.emitter.serialization_helper import pre_json_transform from datahub.metadata.com.linkedin.pegasus2avro.common import GlossaryTerms from datahub.metadata.schema_classes import ( + AssertionKeyClass, AuditStampClass, ContainerKeyClass, DatasetKeyClass, @@ -146,6 +147,14 @@ def container_urn_to_key(guid: str) -> Optional[ContainerKeyClass]: return None +def assertion_urn_to_key(guid: str) -> Optional[AssertionKeyClass]: + pattern = r"urn:li:assertion:(.*)" + results = re.search(pattern, guid) + if results is not None: + return AssertionKeyClass(assertionId=results[1]) + return None + + def datahub_guid(obj: dict) -> str: obj_str = json.dumps( pre_json_transform(obj), separators=(",", ":"), sort_keys=True @@ -216,7 +225,6 @@ def make_domain_urn(domain: str) -> str: def make_ml_primary_key_urn(feature_table_name: str, primary_key_name: str) -> str: - return f"urn:li:mlPrimaryKey:({feature_table_name},{primary_key_name})" @@ -224,7 +232,6 @@ def make_ml_feature_urn( feature_table_name: str, feature_name: str, ) -> str: - return f"urn:li:mlFeature:({feature_table_name},{feature_name})" diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index b5889c5bc0767f..70f3bc6efcbb95 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -1135,7 +1135,7 @@ def string_map(input_map: Dict[str, Any]) -> Dict[str, str]: } ) ) - self.save_checkpoint(node_datahub_urn) + self.save_checkpoint(node_datahub_urn, "assertion") dpi_mcp = MetadataChangeProposalWrapper( entityType="assertion", @@ -1435,7 +1435,7 @@ def create_platform_mces( self.config.env, mce_platform_instance, ) - self.save_checkpoint(node_datahub_urn) + self.save_checkpoint(node_datahub_urn, "dataset") meta_aspects: Dict[str, Any] = {} if self.config.enable_meta_mapping and node.meta: @@ -1511,7 +1511,7 @@ def create_platform_mces( self.report.report_workunit(wu) yield wu - def save_checkpoint(self, node_datahub_urn: str) -> None: + def save_checkpoint(self, node_datahub_urn: str, entity_type: str) -> None: if self.is_stateful_ingestion_configured(): cur_checkpoint = self.get_current_checkpoint( self.get_default_ingestion_job_id() @@ -1522,7 +1522,14 @@ def save_checkpoint(self, node_datahub_urn: str) -> None: checkpoint_state = cast( BaseSQLAlchemyCheckpointState, cur_checkpoint.state ) - checkpoint_state.add_table_urn(node_datahub_urn) + if entity_type == "dataset": + checkpoint_state.add_table_urn(node_datahub_urn) + elif entity_type == "assertion": + checkpoint_state.add_assertion_urn(node_datahub_urn) + else: + logger.error( + f"Can not save Unknown entity {entity_type} to checkpoint." + ) def extract_query_tag_aspects( self, diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py index 49ef4b69736410..da798273b53d86 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py @@ -3,6 +3,7 @@ import pydantic from datahub.emitter.mce_builder import ( + assertion_urn_to_key, container_urn_to_key, dataset_urn_to_key, make_container_urn, @@ -21,6 +22,7 @@ class BaseSQLAlchemyCheckpointState(CheckpointStateBase): encoded_table_urns: List[str] = pydantic.Field(default_factory=list) encoded_view_urns: List[str] = pydantic.Field(default_factory=list) encoded_container_urns: List[str] = pydantic.Field(default_factory=list) + encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) @staticmethod def _get_separator() -> str: @@ -42,6 +44,13 @@ def _get_container_lightweight_repr(container_urn: str) -> str: assert key is not None return f"{key.guid}" + @staticmethod + def _get_assertion_lightweight_repr(assertion_urn: str) -> str: + """Reduces the amount of text in the URNs for smaller state footprint.""" + key = assertion_urn_to_key(assertion_urn) + assert key is not None + return f"{key.assertionId}" + @staticmethod def _get_dataset_urns_not_in( encoded_urns_1: List[str], encoded_urns_2: List[str] @@ -88,6 +97,11 @@ def add_table_urn(self, table_urn: str) -> None: def add_view_urn(self, view_urn: str) -> None: self.encoded_view_urns.append(self._get_lightweight_repr(view_urn)) + def add_assertion_urn(self, assertion_urn: str) -> None: + self.encoded_assertion_urns.append( + self._get_assertion_lightweight_repr(assertion_urn) + ) + def add_container_guid(self, container_urn: str) -> None: self.encoded_container_urns.append( self._get_container_lightweight_repr(container_urn) From 1fd80fd42c705c232b8aa563b8a9fa3de95f3dad Mon Sep 17 00:00:00 2001 From: MugdhaHardikar-GSLab Date: Mon, 18 Jul 2022 18:21:46 +0530 Subject: [PATCH 02/10] fix(dbt): add integration test for stateful with tests --- .../dbt/dbt_stateful_tests_golden.json | 3307 +++++++++++++++++ .../tests/integration/dbt/test_dbt.py | 71 + 2 files changed, 3378 insertions(+) create mode 100644 metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json diff --git a/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json new file mode 100644 index 00000000000000..56d2b7e1740f59 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json @@ -0,0 +1,3307 @@ +[ +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_customers.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_payments.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n \n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/staging/stg_orders.sql", + "catalog_type": "view", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "stg_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.stg_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_customers.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_customers", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_orders.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_orders", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "user_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"seed\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "seed", + "materialization": "seed", + "dbt_file_path": "seeds/raw_payments.csv", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "raw_payments", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "seed.jaffle_shop.raw_payments", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "payment_method", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/customers.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "customers", + "qualifiedName": null, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.customers", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for a customer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's first name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "last_name", + "jsonPath": null, + "nullable": false, + "description": "Customer's last name. PII.", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "first_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's first order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "most_recent_order", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) of a customer's most recent order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "number_of_orders", + "jsonPath": null, + "nullable": false, + "description": "Count of the number of orders a customer has placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_lifetime_value", + "jsonPath": null, + "nullable": false, + "description": null, + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "with customers as (\n\n select * from {{ ref('stg_customers') }}\n\n),\n\norders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\ncustomer_orders as (\n\n select\n customer_id,\n\n min(order_date) as first_order,\n max(order_date) as most_recent_order,\n count(order_id) as number_of_orders\n from orders\n\n group by customer_id\n\n),\n\ncustomer_payments as (\n\n select\n orders.customer_id,\n sum(amount) as total_amount\n\n from payments\n\n left join orders on\n payments.order_id = orders.order_id\n\n group by orders.customer_id\n\n),\n\nfinal as (\n\n select\n customers.customer_id,\n customers.first_name,\n customers.last_name,\n customer_orders.first_order,\n customer_orders.most_recent_order,\n customer_orders.number_of_orders,\n customer_payments.total_amount as customer_lifetime_value\n\n from customers\n\n left join customer_orders\n on customers.customer_id = customer_orders.customer_id\n\n left join customer_payments\n on customers.customer_id = customer_payments.customer_id\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/orders.sql", + "catalog_type": "table", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", + "manifest_version": "1.1.0", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.1.0" + }, + "externalUrl": null, + "name": "orders", + "qualifiedName": null, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.jaffle_shop.orders", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "order_id", + "jsonPath": null, + "nullable": false, + "description": "This is a unique identifier for an order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "customer_id", + "jsonPath": null, + "nullable": false, + "description": "Foreign key to the customers table", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "order_date", + "jsonPath": null, + "nullable": false, + "description": "Date (UTC) that the order was placed", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "status", + "jsonPath": null, + "nullable": false, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "credit_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by credit card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "coupon_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by coupon", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "bank_transfer_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by bank transfer", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "gift_card_amount", + "jsonPath": null, + "nullable": false, + "description": "Amount of the order (AUD) paid for by gift card", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + }, + { + "fieldPath": "amount", + "jsonPath": null, + "nullable": false, + "description": "Total amount (AUD) of the order", + "created": null, + "lastModified": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "FLOAT64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "isPartitioningKey": null, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{% set payment_methods = ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] %}\n\nwith orders as (\n\n select * from {{ ref('stg_orders') }}\n\n),\n\npayments as (\n\n select * from {{ ref('stg_payments') }}\n\n),\n\norder_payments as (\n\n select\n order_id,\n\n {% for payment_method in payment_methods -%}\n sum(case when payment_method = '{{ payment_method }}' then amount else 0 end) as {{ payment_method }}_amount,\n {% endfor -%}\n\n sum(amount) as total_amount\n\n from payments\n\n group by order_id\n\n),\n\nfinal as (\n\n select\n orders.order_id,\n orders.customer_id,\n orders.order_date,\n orders.status,\n\n {% for payment_method in payment_methods -%}\n\n order_payments.{{ payment_method }}_amount,\n\n {% endfor -%}\n\n order_payments.total_amount as amount\n\n from orders\n\n\n left join order_payments\n on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final", + "viewLanguage": "SQL" + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"assert_total_payment_amount_is_positive\", \"nativeParameters\": {}, \"logic\": \"-- Refunds have a negative amount, so the total amount should always be >= 0.\\n-- Therefore return records where this isn't true to make the test fail\\nselect\\n order_id,\\n sum(amount) as total_amount\\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\\ngroup by 1\\nhaving not(total_amount >= 0)\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_amount\", \"nativeParameters\": {\"column_name\": \"amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_credit_card_amount\", \"nativeParameters\": {\"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),coupon_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_coupon_amount\", \"nativeParameters\": {\"column_name\": \"coupon_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),bank_transfer_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_bank_transfer_amount\", \"nativeParameters\": {\"column_name\": \"bank_transfer_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:dbt\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),gift_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_gift_card_amount\", \"nativeParameters\": {\"column_name\": \"gift_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131075, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:da743330013b7e3e3707ac6e526ab408\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131077, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131073, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:2887b9c826e0be6296a37833bdc380bd\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565131058, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b052a324c05327985f3b579a19ad7579\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565137668, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {\"message\": \"Database Error in test dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2 (models/schema.yml)\\n No matching signature for operator = for argument types: INT64, STRING. Supported signature: ANY = ANY at [46:25]\\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_e42202dc29e1149de0f5c3966219796d.sql\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565132560, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:44519aa345bf3ea896179f9f352ae946\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565137668, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:1c217b7587a0cad47a07a09bfe154055\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"FAILURE\", \"nativeResults\": {\"message\": \"Database Error in test dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0 (models/schema.yml)\\n No matching signature for operator = for argument types: FLOAT64, STRING. Supported signature: ANY = ANY at [36:25]\\n compiled SQL at target/run/jaffle_shop/models/schema.yml/dbt_expectations_expect_column_fdf581b1071168614662824120d65b90.sql\"}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133585, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:bbd78a070092f54313153abec49f6f31\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133595, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:ca065a99637630468f688717590beeab\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565133591, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:52d06197762e3608d94609e96f03a0a7\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134031, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134482, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134485, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134493, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565134966, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135368, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135377, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:c1eebc71f36690e4523adca30314e927\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135510, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135510, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565135836, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136269, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136230, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:54bac90e6785bdefd8685ebf8814c429\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136395, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:815963e1332b46a203504ba46ebfab24\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "assertion", + "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "value": "{\"timestampMillis\": 1655565136719, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"runId\": \"c7a6b778-0e0f-4789-b567-ca7e124a6840\", \"assertionUrn\": \"urn:li:assertion:fac27f352406b941125292413afa8096\", \"asserteeUrn\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"status\": \"COMPLETE\", \"result\": {\"type\": \"SUCCESS\", \"nativeResults\": {}}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "test_pipeline", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 23a214778f04b8..ecc0e1c5123b9f 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -470,3 +470,74 @@ def test_dbt_tests(pytestconfig, tmp_path, mock_time, **kwargs): golden_path=golden_path, ignore_paths=[], ) + + +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_stateful_tests(pytestconfig, tmp_path, mock_time, mock_datahub_graph): + + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + output_file = tmp_path / "dbt_stateful_tests.json" + golden_path = test_resources_dir / "dbt_stateful_tests_golden.json" + manifest_path = str((test_resources_dir / "jaffle_shop_manifest.json").resolve()) + catalog_path = str((test_resources_dir / "jaffle_shop_catalog.json").resolve()) + test_results_path = str( + (test_resources_dir / "jaffle_shop_test_results.json").resolve() + ) + + stateful_config = { + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + } + + scd: Dict[str, Any] = { + "manifest_path": manifest_path, + "catalog_path": catalog_path, + "test_results_path": test_results_path, + "target_platform": "postgres", + "load_schemas": True, + # This will bypass check in get_workunits function of dbt.py + "write_semantics": "OVERRIDE", + "owner_extraction_pattern": r"^@(?P(.*))", + # enable stateful ingestion + **stateful_config, + } + + pipeline_config_dict: Dict[str, Any] = { + "source": { + "type": "dbt", + "config": scd, + }, + "sink": { + # we are not really interested in the resulting events for this test + "type": "file", + "config": {"filename": str(output_file)}, + }, + "pipeline_name": "statefulpipeline", + "run_id": "test_pipeline", + } + + with patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + pipeline = Pipeline.create(pipeline_config_dict) + pipeline.run() + pipeline.raise_from_status() + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_file, + golden_path=golden_path, + ignore_paths=[], + ) + # Do the first run of the pipeline and get the default job's checkpoint. + # pipeline_run1 = run_and_get_pipeline(pipeline_config_dict) + # print(pipeline_run1) From f6325fa2aee5cd6e451496425d6a7f2640422b12 Mon Sep 17 00:00:00 2001 From: MugdhaHardikar-GSLab Date: Mon, 18 Jul 2022 18:33:31 +0530 Subject: [PATCH 03/10] remove unwanted comments --- metadata-ingestion/tests/integration/dbt/test_dbt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index ecc0e1c5123b9f..2978f7a6165ced 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -538,6 +538,4 @@ def test_dbt_stateful_tests(pytestconfig, tmp_path, mock_time, mock_datahub_grap golden_path=golden_path, ignore_paths=[], ) - # Do the first run of the pipeline and get the default job's checkpoint. - # pipeline_run1 = run_and_get_pipeline(pipeline_config_dict) - # print(pipeline_run1) + From bab7dec7ad20c06c542bb7c4b9ee277f8195ee29 Mon Sep 17 00:00:00 2001 From: MugdhaHardikar-GSLab Date: Mon, 18 Jul 2022 21:23:55 +0530 Subject: [PATCH 04/10] lintFix --- metadata-ingestion/tests/integration/dbt/test_dbt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 2978f7a6165ced..9cfa9ba7bb7fe8 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -538,4 +538,3 @@ def test_dbt_stateful_tests(pytestconfig, tmp_path, mock_time, mock_datahub_grap golden_path=golden_path, ignore_paths=[], ) - From 89d3d83f5f208ec2bff204e1a8d284a41e792e19 Mon Sep 17 00:00:00 2001 From: MugdhaHardikar-GSLab Date: Wed, 20 Jul 2022 10:02:21 +0530 Subject: [PATCH 05/10] use utitlies/urn instead of creating new assertion parser funstion --- metadata-ingestion/src/datahub/emitter/mce_builder.py | 9 --------- .../datahub/ingestion/source/state/sql_common_state.py | 7 ++++--- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index 5be422e9dc400a..cdbde24ac4d88c 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -15,7 +15,6 @@ from datahub.emitter.serialization_helper import pre_json_transform from datahub.metadata.com.linkedin.pegasus2avro.common import GlossaryTerms from datahub.metadata.schema_classes import ( - AssertionKeyClass, AuditStampClass, ContainerKeyClass, DatasetKeyClass, @@ -147,14 +146,6 @@ def container_urn_to_key(guid: str) -> Optional[ContainerKeyClass]: return None -def assertion_urn_to_key(guid: str) -> Optional[AssertionKeyClass]: - pattern = r"urn:li:assertion:(.*)" - results = re.search(pattern, guid) - if results is not None: - return AssertionKeyClass(assertionId=results[1]) - return None - - def datahub_guid(obj: dict) -> str: obj_str = json.dumps( pre_json_transform(obj), separators=(",", ":"), sort_keys=True diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py index da798273b53d86..ff8d7cec08e44a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py @@ -3,13 +3,13 @@ import pydantic from datahub.emitter.mce_builder import ( - assertion_urn_to_key, container_urn_to_key, dataset_urn_to_key, make_container_urn, make_dataset_urn, ) from datahub.ingestion.source.state.checkpoint import CheckpointStateBase +from datahub.utilities.urns.urn import Urn class BaseSQLAlchemyCheckpointState(CheckpointStateBase): @@ -47,9 +47,10 @@ def _get_container_lightweight_repr(container_urn: str) -> str: @staticmethod def _get_assertion_lightweight_repr(assertion_urn: str) -> str: """Reduces the amount of text in the URNs for smaller state footprint.""" - key = assertion_urn_to_key(assertion_urn) + urn = Urn.create_from_string(assertion_urn) + key = urn.get_entity_id_as_string() assert key is not None - return f"{key.assertionId}" + return key @staticmethod def _get_dataset_urns_not_in( From 14e90a6411fa7840ea93d02525d1003620efd236 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Fri, 22 Jul 2022 19:35:28 +0530 Subject: [PATCH 06/10] Store state in DbtCheckpointState --- .../src/datahub/ingestion/source/dbt.py | 183 ++++++++++++++---- .../source/state/sql_common_state.py | 59 ++---- .../src/datahub/utilities/check_point_util.py | 39 ++++ .../tests/integration/dbt/test_dbt.py | 115 ++++++++++- .../tests/test_helpers/state_helpers.py | 5 + 5 files changed, 313 insertions(+), 88 deletions(-) create mode 100644 metadata-ingestion/src/datahub/utilities/check_point_util.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index 70f3bc6efcbb95..7ff6755fd3c178 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -18,12 +18,14 @@ from urllib.parse import urlparse import dateutil.parser +import pydantic import requests from pydantic import BaseModel, validator from pydantic.fields import Field from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.emitter import mce_builder +from datahub.emitter.mce_builder import make_assertion_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( @@ -44,7 +46,7 @@ SPARK_SQL_TYPES_MAP, resolve_postgres_modified_type, ) -from datahub.ingestion.source.state.checkpoint import Checkpoint +from datahub.ingestion.source.state.checkpoint import Checkpoint, CheckpointStateBase from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, ) @@ -107,7 +109,9 @@ UpstreamLineageClass, ViewPropertiesClass, ) +from datahub.utilities.check_point_util import CheckpointStateUtil from datahub.utilities.mapping import Constants, OperationProcessor +from datahub.utilities.urns.urn import Urn logger = logging.getLogger(__name__) DBT_PLATFORM = "dbt" @@ -123,6 +127,65 @@ class DBTStatefulIngestionConfig(StatefulIngestionConfig): remove_stale_metadata: bool = True +class DbtCheckpointState(CheckpointStateBase): + """ + Class for representing the checkpoint state for DBT sources. + Stores all nodes and assertions being ingested and is used to remove any stale entities. + """ + + encoded_node_urns: List[str] = pydantic.Field(default_factory=list) + encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) + + @staticmethod + def _get_assertion_lightweight_repr(assertion_urn: str) -> str: + """Reduces the amount of text in the URNs for smaller state footprint.""" + urn = Urn.create_from_string(assertion_urn) + key = urn.get_entity_id_as_string() + assert key is not None + return key + + def add_assertion_urn(self, assertion_urn: str) -> None: + self.encoded_assertion_urns.append( + self._get_assertion_lightweight_repr(assertion_urn) + ) + + def get_assertion_urns_not_in( + self, checkpoint: "DbtCheckpointState" + ) -> Iterable[str]: + """ + Dbt assertion are mapped to DataHub assertion concept + """ + difference = CheckpointStateUtil.get_encoded_urns_not_in( + self.encoded_assertion_urns, checkpoint.encoded_assertion_urns + ) + for key in difference: + yield make_assertion_urn(key) + + def get_node_urns_not_in(self, checkpoint: "DbtCheckpointState") -> Iterable[str]: + """ + Dbt node are mapped to DataHub dataset concept + """ + yield from CheckpointStateUtil.get_dataset_urns_not_in( + self.encoded_node_urns, checkpoint.encoded_node_urns + ) + + def add_node_urn(self, node_urn: str) -> None: + self.encoded_node_urns.append( + CheckpointStateUtil.get_dataset_lightweight_repr(node_urn) + ) + + def set_checkpoint_urn(self, urn: str, entity_type: str) -> None: + supported_entities = { + "dataset": self.add_node_urn, + "assertion": self.add_assertion_urn, + } + + if supported_entities.get(entity_type) is None: + logger.error(f"Can not save Unknown entity {entity_type} to checkpoint.") + + supported_entities[entity_type](urn) + + @dataclass class DBTSourceReport(StatefulIngestionReport): soft_deleted_stale_entities: List[str] = field(default_factory=list) @@ -989,15 +1052,51 @@ def __init__(self, config: DBTConfig, ctx: PipelineContext, platform: str): self.config.owner_extraction_pattern ) + def get_last_dbt_checkpoint( + self, job_id: JobId, checkpoint_state_class: Type[CheckpointStateBase] + ) -> Optional[Checkpoint]: + + last_checkpoint: Optional[Checkpoint] = cast(Checkpoint, None) + is_conversion_required: bool = False + try: + # Best-case that last checkpoint state is DbtCheckpointState + last_checkpoint = self.get_last_checkpoint( + self.get_default_ingestion_job_id(), DbtCheckpointState + ) + except Exception: + # Backward compatibility for old dbt ingestion source which was saving dbt-nodes in + # BaseSQLAlchemyCheckpointState + last_checkpoint = self.get_last_checkpoint( + self.get_default_ingestion_job_id(), BaseSQLAlchemyCheckpointState + ) + logger.debug("Found BaseSQLAlchemyCheckpointState as checkpoint state") + is_conversion_required = True + + if last_checkpoint is None: + return None + + if is_conversion_required: + # Map the BaseSQLAlchemyCheckpointState to DbtCheckpointState + dbt_checkpoint_state: DbtCheckpointState = DbtCheckpointState() + dbt_checkpoint_state.encoded_node_urns = ( + cast(BaseSQLAlchemyCheckpointState, last_checkpoint.state) + ).encoded_table_urns + # Old dbt source was not supporting the assertion + dbt_checkpoint_state.encoded_assertion_urns = [] + last_checkpoint.state = dbt_checkpoint_state + + return last_checkpoint + # TODO: Consider refactoring this logic out for use across sources as it is leading to a significant amount of # code duplication. def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: - last_checkpoint = self.get_last_checkpoint( - self.get_default_ingestion_job_id(), BaseSQLAlchemyCheckpointState + last_checkpoint: Optional[Checkpoint] = self.get_last_dbt_checkpoint( + self.get_default_ingestion_job_id(), DbtCheckpointState ) cur_checkpoint = self.get_current_checkpoint( self.get_default_ingestion_job_id() ) + if ( self.config.stateful_ingestion and self.config.stateful_ingestion.remove_stale_metadata @@ -1008,7 +1107,7 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: ): logger.debug("Checking for stale entity removal.") - def soft_delete_item(urn: str, type: str) -> Iterable[MetadataWorkUnit]: + def soft_delete_item(urn: str, type: str) -> MetadataWorkUnit: logger.info(f"Soft-deleting stale entity of type {type} - {urn}.") mcp = MetadataChangeProposalWrapper( @@ -1021,19 +1120,28 @@ def soft_delete_item(urn: str, type: str) -> Iterable[MetadataWorkUnit]: wu = MetadataWorkUnit(id=f"soft-delete-{type}-{urn}", mcp=mcp) self.report.report_workunit(wu) self.report.report_stale_entity_soft_deleted(urn) - yield wu + return wu - last_checkpoint_state = cast( - BaseSQLAlchemyCheckpointState, last_checkpoint.state - ) - cur_checkpoint_state = cast( - BaseSQLAlchemyCheckpointState, cur_checkpoint.state - ) + last_checkpoint_state = cast(DbtCheckpointState, last_checkpoint.state) + cur_checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) - for table_urn in last_checkpoint_state.get_table_urns_not_in( - cur_checkpoint_state - ): - yield from soft_delete_item(table_urn, "dataset") + soft_delete_urn: Dict = { + "dataset": [ + node_urn + for node_urn in last_checkpoint_state.get_node_urns_not_in( + cur_checkpoint_state + ) + ], + "assertion": [ + assertion_urn + for assertion_urn in last_checkpoint_state.get_assertion_urns_not_in( + cur_checkpoint_state + ) + ], + } + for entity_type in soft_delete_urn: + for urn in soft_delete_urn[entity_type]: + yield soft_delete_item(urn, entity_type) def load_file_as_json(self, uri: str) -> Any: if re.match("^https?://", uri): @@ -1389,10 +1497,12 @@ def remove_duplicate_urns_from_checkpoint_state(self) -> None: ) if cur_checkpoint is not None: - # Utilizing BaseSQLAlchemyCheckpointState class to save state - checkpoint_state = cast(BaseSQLAlchemyCheckpointState, cur_checkpoint.state) - checkpoint_state.encoded_table_urns = list( - set(checkpoint_state.encoded_table_urns) + checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) + checkpoint_state.encoded_node_urns = list( + set(checkpoint_state.encoded_node_urns) + ) + checkpoint_state.encoded_assertion_urns = list( + set(checkpoint_state.encoded_assertion_urns) ) def create_platform_mces( @@ -1511,25 +1621,21 @@ def create_platform_mces( self.report.report_workunit(wu) yield wu - def save_checkpoint(self, node_datahub_urn: str, entity_type: str) -> None: - if self.is_stateful_ingestion_configured(): - cur_checkpoint = self.get_current_checkpoint( - self.get_default_ingestion_job_id() - ) + def save_checkpoint(self, urn: str, entity_type: str) -> None: + # if stateful ingestion is not configured then return + if not self.is_stateful_ingestion_configured(): + return - if cur_checkpoint is not None: - # Utilizing BaseSQLAlchemyCheckpointState class to save state - checkpoint_state = cast( - BaseSQLAlchemyCheckpointState, cur_checkpoint.state - ) - if entity_type == "dataset": - checkpoint_state.add_table_urn(node_datahub_urn) - elif entity_type == "assertion": - checkpoint_state.add_assertion_urn(node_datahub_urn) - else: - logger.error( - f"Can not save Unknown entity {entity_type} to checkpoint." - ) + cur_checkpoint = self.get_current_checkpoint( + self.get_default_ingestion_job_id() + ) + # if no checkpoint found then return + if cur_checkpoint is None: + return + + # Cast and set the state + checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) + checkpoint_state.set_checkpoint_urn(urn, entity_type) def extract_query_tag_aspects( self, @@ -1882,8 +1988,7 @@ def create_checkpoint(self, job_id: JobId) -> Optional[Checkpoint]: platform_instance_id=self.get_platform_instance_id(), run_id=self.ctx.run_id, config=self.config, - # Reusing BaseSQLAlchemyCheckpointState as it has needed functionality to support statefulness of DBT - state=BaseSQLAlchemyCheckpointState(), + state=DbtCheckpointState(), ) return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py index ff8d7cec08e44a..65db34c23d4c91 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py @@ -2,14 +2,9 @@ import pydantic -from datahub.emitter.mce_builder import ( - container_urn_to_key, - dataset_urn_to_key, - make_container_urn, - make_dataset_urn, -) +from datahub.emitter.mce_builder import container_urn_to_key, make_container_urn from datahub.ingestion.source.state.checkpoint import CheckpointStateBase -from datahub.utilities.urns.urn import Urn +from datahub.utilities.check_point_util import CheckpointStateUtil class BaseSQLAlchemyCheckpointState(CheckpointStateBase): @@ -24,18 +19,10 @@ class BaseSQLAlchemyCheckpointState(CheckpointStateBase): encoded_container_urns: List[str] = pydantic.Field(default_factory=list) encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) - @staticmethod - def _get_separator() -> str: - # Unique small string not allowed in URNs. - return "||" - @staticmethod def _get_lightweight_repr(dataset_urn: str) -> str: """Reduces the amount of text in the URNs for smaller state footprint.""" - SEP = BaseSQLAlchemyCheckpointState._get_separator() - key = dataset_urn_to_key(dataset_urn) - assert key is not None - return f"{key.platform}{SEP}{key.name}{SEP}{key.origin}" + return CheckpointStateUtil.get_dataset_lightweight_repr(dataset_urn) @staticmethod def _get_container_lightweight_repr(container_urn: str) -> str: @@ -44,44 +31,35 @@ def _get_container_lightweight_repr(container_urn: str) -> str: assert key is not None return f"{key.guid}" - @staticmethod - def _get_assertion_lightweight_repr(assertion_urn: str) -> str: - """Reduces the amount of text in the URNs for smaller state footprint.""" - urn = Urn.create_from_string(assertion_urn) - key = urn.get_entity_id_as_string() - assert key is not None - return key - - @staticmethod - def _get_dataset_urns_not_in( - encoded_urns_1: List[str], encoded_urns_2: List[str] - ) -> Iterable[str]: - difference = set(encoded_urns_1) - set(encoded_urns_2) - for encoded_urn in difference: - platform, name, env = encoded_urn.split( - BaseSQLAlchemyCheckpointState._get_separator() - ) - yield make_dataset_urn(platform, name, env) - @staticmethod def _get_container_urns_not_in( encoded_urns_1: List[str], encoded_urns_2: List[str] ) -> Iterable[str]: - difference = set(encoded_urns_1) - set(encoded_urns_2) + difference = CheckpointStateUtil.get_encoded_urns_not_in( + encoded_urns_1, encoded_urns_2 + ) for guid in difference: yield make_container_urn(guid) def get_table_urns_not_in( self, checkpoint: "BaseSQLAlchemyCheckpointState" ) -> Iterable[str]: - yield from self._get_dataset_urns_not_in( + """ + Tables are mapped to DataHub dataset concept + """ + + yield from CheckpointStateUtil.get_dataset_urns_not_in( self.encoded_table_urns, checkpoint.encoded_table_urns ) def get_view_urns_not_in( self, checkpoint: "BaseSQLAlchemyCheckpointState" ) -> Iterable[str]: - yield from self._get_dataset_urns_not_in( + """ + Views are mapped to DataHub dataset concept + """ + + yield from CheckpointStateUtil.get_dataset_urns_not_in( self.encoded_view_urns, checkpoint.encoded_view_urns ) @@ -98,11 +76,6 @@ def add_table_urn(self, table_urn: str) -> None: def add_view_urn(self, view_urn: str) -> None: self.encoded_view_urns.append(self._get_lightweight_repr(view_urn)) - def add_assertion_urn(self, assertion_urn: str) -> None: - self.encoded_assertion_urns.append( - self._get_assertion_lightweight_repr(assertion_urn) - ) - def add_container_guid(self, container_urn: str) -> None: self.encoded_container_urns.append( self._get_container_lightweight_repr(container_urn) diff --git a/metadata-ingestion/src/datahub/utilities/check_point_util.py b/metadata-ingestion/src/datahub/utilities/check_point_util.py new file mode 100644 index 00000000000000..4208b859b615c6 --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/check_point_util.py @@ -0,0 +1,39 @@ +from typing import Iterable, List, Set + +from datahub.emitter.mce_builder import dataset_urn_to_key, make_dataset_urn + + +class CheckpointStateUtil: + """ + A utility class to provide common functionalities around Urn and CheckPointState of different DataHub entities + """ + + @staticmethod + def get_separator() -> str: + # Unique small string not allowed in URNs. + return "||" + + @staticmethod + def get_encoded_urns_not_in( + encoded_urns_1: List[str], encoded_urns_2: List[str] + ) -> Set[str]: + + return set(encoded_urns_1) - set(encoded_urns_2) + + @staticmethod + def get_dataset_lightweight_repr(dataset_urn: str) -> str: + SEP = CheckpointStateUtil.get_separator() + key = dataset_urn_to_key(dataset_urn) + assert key is not None + return f"{key.platform}{SEP}{key.name}{SEP}{key.origin}" + + @staticmethod + def get_dataset_urns_not_in( + encoded_urns_1: List[str], encoded_urns_2: List[str] + ) -> Iterable[str]: + difference = CheckpointStateUtil.get_encoded_urns_not_in( + encoded_urns_1, encoded_urns_2 + ) + for encoded_urn in difference: + platform, name, env = encoded_urn.split(CheckpointStateUtil.get_separator()) + yield make_dataset_urn(platform, name, env) diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 9cfa9ba7bb7fe8..d90304602c37c6 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -1,5 +1,5 @@ from os import PathLike -from typing import Any, Dict, Optional, Union, cast +from typing import Any, Dict, Optional, Type, Union, cast from unittest.mock import patch import pytest @@ -7,14 +7,16 @@ from freezegun import freeze_time from datahub.configuration.common import DynamicTypedConfig +from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import JobId from datahub.ingestion.run.pipeline import Pipeline, PipelineConfig, SourceConfig -from datahub.ingestion.source.dbt import DBTConfig, DBTSource -from datahub.ingestion.source.state.checkpoint import Checkpoint +from datahub.ingestion.source.dbt import DbtCheckpointState, DBTConfig, DBTSource +from datahub.ingestion.source.state.checkpoint import Checkpoint, CheckpointStateBase from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, ) from tests.test_helpers import mce_helpers from tests.test_helpers.state_helpers import ( + create_pipeline, run_and_get_pipeline, validate_all_providers_have_committed_successfully, ) @@ -401,9 +403,9 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): # Perform all assertions on the states. The deleted table should not be # part of the second state - state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state) - state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state) - difference_urns = list(state1.get_table_urns_not_in(state2)) + state1 = cast(DbtCheckpointState, checkpoint1.state) + state2 = cast(DbtCheckpointState, checkpoint2.state) + difference_urns = list(state1.get_node_urns_not_in(state2)) assert len(difference_urns) == 2 @@ -429,6 +431,107 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): ) +@pytest.mark.integration +@freeze_time(FROZEN_TIME) +def test_dbt_state_backward_compatibility( + pytestconfig, tmp_path, mock_time, mock_datahub_graph +): + test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + + manifest_path = "{}/dbt_manifest.json".format(test_resources_dir) + catalog_path = "{}/dbt_catalog.json".format(test_resources_dir) + sources_path = "{}/dbt_sources.json".format(test_resources_dir) + + stateful_config = { + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + } + + scd_config: Dict[str, Any] = { + "manifest_path": manifest_path, + "catalog_path": catalog_path, + "sources_path": sources_path, + "target_platform": "postgres", + "load_schemas": True, + # This will bypass check in get_workunits function of dbt.py + "write_semantics": "OVERRIDE", + "owner_extraction_pattern": r"^@(?P(.*))", + # enable stateful ingestion + **stateful_config, + } + + pipeline_config_dict: Dict[str, Any] = { + "source": { + "type": "dbt", + "config": scd_config, + }, + "sink": { + # we are not really interested in the resulting events for this test + "type": "console" + }, + "pipeline_name": "statefulpipeline", + } + + with patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + + pipeline = create_pipeline(pipeline_config_dict) + + dbt_source = cast(DBTSource, pipeline.source) + + def get_fake_base_sql_alchemy_checkpoint_state( + job_id: JobId, checkpoint_state_class: Type[CheckpointStateBase] + ) -> Optional[Checkpoint]: + if checkpoint_state_class is DbtCheckpointState: + raise Exception( + "DBT source will call this function again with BaseSQLAlchemyCheckpointState" + ) + + sql_state = BaseSQLAlchemyCheckpointState() + + urn1 = "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)" + urn2 = ( + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)" + ) + + sql_state.add_table_urn(urn1) + sql_state.add_table_urn(urn2) + + assert dbt_source.ctx.pipeline_name is not None + + return Checkpoint( + job_name=dbt_source.get_default_ingestion_job_id(), + pipeline_name=dbt_source.ctx.pipeline_name, + platform_instance_id=dbt_source.get_platform_instance_id(), + run_id=dbt_source.ctx.run_id, + config=dbt_source.config, + state=sql_state, + ) + + # Set fake method to return BaseSQLAlchemyCheckpointState + dbt_source.get_last_checkpoint = get_fake_base_sql_alchemy_checkpoint_state # type: ignore[assignment] + + last_checkpoint = dbt_source.get_last_dbt_checkpoint( + dbt_source.get_default_ingestion_job_id(), DbtCheckpointState + ) + # Our fake method is returning BaseSQLAlchemyCheckpointState,however it should get converted to DbtCheckpointState + assert last_checkpoint is not None and isinstance( + last_checkpoint.state, DbtCheckpointState + ) + + pipeline.run() + pipeline.raise_from_status() + + @pytest.mark.integration @freeze_time(FROZEN_TIME) def test_dbt_tests(pytestconfig, tmp_path, mock_time, **kwargs): diff --git a/metadata-ingestion/tests/test_helpers/state_helpers.py b/metadata-ingestion/tests/test_helpers/state_helpers.py index 4f26b6d87d7ae0..4da41988a97ac6 100644 --- a/metadata-ingestion/tests/test_helpers/state_helpers.py +++ b/metadata-ingestion/tests/test_helpers/state_helpers.py @@ -35,6 +35,11 @@ def run_and_get_pipeline(pipeline_config_dict: Dict[str, Any]) -> Pipeline: return pipeline +def create_pipeline(pipeline_config_dict: Dict[str, Any]) -> Pipeline: + pipeline = Pipeline.create(pipeline_config_dict) + return pipeline + + @pytest.fixture def mock_datahub_graph(): class MockDataHubGraphContext: From 288259e6b1254a979e6ff027942db32ac11aec8d Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Mon, 25 Jul 2022 14:30:44 +0530 Subject: [PATCH 07/10] remove tuple from urn assignment --- metadata-ingestion/tests/integration/dbt/test_dbt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index d90304602c37c6..33c0357248e7e4 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -499,9 +499,7 @@ def get_fake_base_sql_alchemy_checkpoint_state( sql_state = BaseSQLAlchemyCheckpointState() urn1 = "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)" - urn2 = ( - "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)" - ) + urn2 = "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)" sql_state.add_table_urn(urn1) sql_state.add_table_urn(urn2) From 13c6b89b6b608a2d4cdf5d39733ac8a4ebc1fa09 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Wed, 3 Aug 2022 11:26:55 +0530 Subject: [PATCH 08/10] incorporate review comments --- .../src/datahub/ingestion/source/dbt.py | 64 +---------------- .../ingestion/source/state/dbt_state.py | 70 +++++++++++++++++++ .../tests/integration/dbt/test_dbt.py | 3 +- 3 files changed, 73 insertions(+), 64 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index 891e73866c91f1..c6e9e9d803b506 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -18,14 +18,12 @@ from urllib.parse import urlparse import dateutil.parser -import pydantic import requests from pydantic import BaseModel, validator from pydantic.fields import Field from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.emitter import mce_builder -from datahub.emitter.mce_builder import make_assertion_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( @@ -49,6 +47,7 @@ resolve_trino_modified_type, ) from datahub.ingestion.source.state.checkpoint import Checkpoint, CheckpointStateBase +from datahub.ingestion.source.state.dbt_state import DbtCheckpointState from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, ) @@ -111,9 +110,7 @@ UpstreamLineageClass, ViewPropertiesClass, ) -from datahub.utilities.check_point_util import CheckpointStateUtil from datahub.utilities.mapping import Constants, OperationProcessor -from datahub.utilities.urns.urn import Urn logger = logging.getLogger(__name__) DBT_PLATFORM = "dbt" @@ -129,65 +126,6 @@ class DBTStatefulIngestionConfig(StatefulIngestionConfig): remove_stale_metadata: bool = True -class DbtCheckpointState(CheckpointStateBase): - """ - Class for representing the checkpoint state for DBT sources. - Stores all nodes and assertions being ingested and is used to remove any stale entities. - """ - - encoded_node_urns: List[str] = pydantic.Field(default_factory=list) - encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) - - @staticmethod - def _get_assertion_lightweight_repr(assertion_urn: str) -> str: - """Reduces the amount of text in the URNs for smaller state footprint.""" - urn = Urn.create_from_string(assertion_urn) - key = urn.get_entity_id_as_string() - assert key is not None - return key - - def add_assertion_urn(self, assertion_urn: str) -> None: - self.encoded_assertion_urns.append( - self._get_assertion_lightweight_repr(assertion_urn) - ) - - def get_assertion_urns_not_in( - self, checkpoint: "DbtCheckpointState" - ) -> Iterable[str]: - """ - Dbt assertion are mapped to DataHub assertion concept - """ - difference = CheckpointStateUtil.get_encoded_urns_not_in( - self.encoded_assertion_urns, checkpoint.encoded_assertion_urns - ) - for key in difference: - yield make_assertion_urn(key) - - def get_node_urns_not_in(self, checkpoint: "DbtCheckpointState") -> Iterable[str]: - """ - Dbt node are mapped to DataHub dataset concept - """ - yield from CheckpointStateUtil.get_dataset_urns_not_in( - self.encoded_node_urns, checkpoint.encoded_node_urns - ) - - def add_node_urn(self, node_urn: str) -> None: - self.encoded_node_urns.append( - CheckpointStateUtil.get_dataset_lightweight_repr(node_urn) - ) - - def set_checkpoint_urn(self, urn: str, entity_type: str) -> None: - supported_entities = { - "dataset": self.add_node_urn, - "assertion": self.add_assertion_urn, - } - - if supported_entities.get(entity_type) is None: - logger.error(f"Can not save Unknown entity {entity_type} to checkpoint.") - - supported_entities[entity_type](urn) - - @dataclass class DBTSourceReport(StatefulIngestionReport): soft_deleted_stale_entities: List[str] = field(default_factory=list) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py new file mode 100644 index 00000000000000..6746619a5de3ce --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py @@ -0,0 +1,70 @@ +import logging +from typing import Iterable, List + +import pydantic + +from datahub.emitter.mce_builder import make_assertion_urn +from datahub.ingestion.source.state.checkpoint import CheckpointStateBase +from datahub.utilities.check_point_util import CheckpointStateUtil +from datahub.utilities.urns.urn import Urn + +logger = logging.getLogger(__name__) + + +class DbtCheckpointState(CheckpointStateBase): + """ + Class for representing the checkpoint state for DBT sources. + Stores all nodes and assertions being ingested and is used to remove any stale entities. + """ + + encoded_node_urns: List[str] = pydantic.Field(default_factory=list) + encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) + + @staticmethod + def _get_assertion_lightweight_repr(assertion_urn: str) -> str: + """Reduces the amount of text in the URNs for smaller state footprint.""" + urn = Urn.create_from_string(assertion_urn) + key = urn.get_entity_id_as_string() + assert key is not None + return key + + def add_assertion_urn(self, assertion_urn: str) -> None: + self.encoded_assertion_urns.append( + self._get_assertion_lightweight_repr(assertion_urn) + ) + + def get_assertion_urns_not_in( + self, checkpoint: "DbtCheckpointState" + ) -> Iterable[str]: + """ + Dbt assertion are mapped to DataHub assertion concept + """ + difference = CheckpointStateUtil.get_encoded_urns_not_in( + self.encoded_assertion_urns, checkpoint.encoded_assertion_urns + ) + for key in difference: + yield make_assertion_urn(key) + + def get_node_urns_not_in(self, checkpoint: "DbtCheckpointState") -> Iterable[str]: + """ + Dbt node are mapped to DataHub dataset concept + """ + yield from CheckpointStateUtil.get_dataset_urns_not_in( + self.encoded_node_urns, checkpoint.encoded_node_urns + ) + + def add_node_urn(self, node_urn: str) -> None: + self.encoded_node_urns.append( + CheckpointStateUtil.get_dataset_lightweight_repr(node_urn) + ) + + def set_checkpoint_urn(self, urn: str, entity_type: str) -> None: + supported_entities = { + "dataset": self.add_node_urn, + "assertion": self.add_assertion_urn, + } + + if supported_entities.get(entity_type) is None: + logger.error(f"Can not save Unknown entity {entity_type} to checkpoint.") + + supported_entities[entity_type](urn) diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 9512f33d0c10da..4d357e5f402f26 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -9,12 +9,13 @@ from datahub.configuration.common import DynamicTypedConfig from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import JobId from datahub.ingestion.run.pipeline import Pipeline, PipelineConfig, SourceConfig -from datahub.ingestion.source.dbt import DbtCheckpointState, DBTConfig, DBTSource +from datahub.ingestion.source.dbt import DBTConfig, DBTSource from datahub.ingestion.source.sql.sql_types import ( TRINO_SQL_TYPES_MAP, resolve_trino_modified_type, ) from datahub.ingestion.source.state.checkpoint import Checkpoint, CheckpointStateBase +from datahub.ingestion.source.state.dbt_state import DbtCheckpointState from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, ) From 130ea14e1716e0b0a159863f045246e5dddb3333 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Wed, 3 Aug 2022 16:35:40 +0530 Subject: [PATCH 09/10] fix golden file --- .../dbt/dbt_stateful_tests_golden.json | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json index 56d2b7e1740f59..eee040cc407a92 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json @@ -33,6 +33,7 @@ "catalog_type": "view", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -209,6 +210,7 @@ "catalog_type": "view", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -405,6 +407,7 @@ "catalog_type": "view", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -601,6 +604,7 @@ "catalog_type": "table", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -754,6 +758,7 @@ "catalog_type": "table", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -927,6 +932,7 @@ "catalog_type": "table", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -1100,6 +1106,7 @@ "catalog_type": "table", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -1374,6 +1381,7 @@ "catalog_type": "table", "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json", "manifest_version": "1.1.0", + "manifest_adapter": "bigquery", "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, @@ -1944,7 +1952,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"assert_total_payment_amount_is_positive\", \"nativeParameters\": {}, \"logic\": \"-- Refunds have a negative amount, so the total amount should always be >= 0.\\n-- Therefore return records where this isn't true to make the test fail\\nselect\\n order_id,\\n sum(amount) as total_amount\\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\\ngroup by 1\\nhaving not(total_amount >= 0)\"}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_ROWS\", \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"assert_total_payment_amount_is_positive\", \"nativeParameters\": {}, \"logic\": \"-- Refunds have a negative amount, so the total amount should always be >= 0.\\n-- Therefore return records where this isn't true to make the test fail\\nselect\\n order_id,\\n sum(amount) as total_amount\\nfrom `calm-pagoda-323403`.`jaffle_shop`.`orders`\\ngroup by 1\\nhaving not(total_amount >= 0)\"}}", "contentType": "application/json" }, "systemMetadata": { @@ -1982,7 +1990,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2020,7 +2028,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('stg_customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2058,7 +2066,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2096,7 +2104,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2134,7 +2142,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('stg_orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2172,7 +2180,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2210,7 +2218,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_stg_payments_payment_id\", \"nativeParameters\": {\"column_name\": \"payment_id\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2248,7 +2256,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD),payment_method)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"credit_card\\\", \\\"coupon\\\", \\\"bank_transfer\\\", \\\"gift_card\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card\", \"nativeParameters\": {\"values\": \"['credit_card', 'coupon', 'bank_transfer', 'gift_card']\", \"column_name\": \"payment_method\", \"model\": \"{{ get_where_subquery(ref('stg_payments')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2286,7 +2294,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2324,7 +2332,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_customers_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2362,7 +2370,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"BETWEEN\", \"parameters\": {\"maxValue\": {\"value\": \"2000000\", \"type\": \"NUMBER\"}, \"minValue\": {\"value\": \"0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_between_customers_customer_id__2000000__0__customer_id_is_not_null__False\", \"nativeParameters\": {\"min_value\": \"0\", \"max_value\": \"2000000\", \"row_condition\": \"customer_id is not null\", \"strictly\": \"False\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2400,7 +2408,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"0\\\", \\\"1\\\", \\\"2\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"dbt_expectations_expect_column_values_to_be_in_set_customers_customer_id__customer_id_is_not_null__0__1__2\", \"nativeParameters\": {\"value_set\": \"['0', '1', '2']\", \"row_condition\": \"customer_id is not null\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('customers')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2438,7 +2446,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"UNIQUE_PROPOTION\", \"operator\": \"EQUAL_TO\", \"parameters\": {\"value\": {\"value\": \"1.0\", \"type\": \"NUMBER\"}}, \"nativeType\": \"unique_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2476,7 +2484,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),order_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_order_id\", \"nativeParameters\": {\"column_name\": \"order_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2514,7 +2522,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_customer_id\", \"nativeParameters\": {\"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2552,7 +2560,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", "contentType": "application/json" }, "systemMetadata": { @@ -2571,7 +2579,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),customer_id)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"_NATIVE_\", \"parameters\": {\"value\": {\"value\": \"null\", \"type\": \"SET\"}}, \"nativeType\": \"relationships_orders_customer_id__customer_id__ref_customers_\", \"nativeParameters\": {\"to\": \"ref('customers')\", \"field\": \"customer_id\", \"column_name\": \"customer_id\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"orders.customer_id referential integrity to customers.customer_id\"}}", "contentType": "application/json" }, "systemMetadata": { @@ -2609,7 +2617,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),status)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"IN\", \"parameters\": {\"value\": {\"value\": \"[\\\"placed\\\", \\\"shipped\\\", \\\"completed\\\", \\\"return_pending\\\", \\\"returned\\\"]\", \"type\": \"SET\"}}, \"nativeType\": \"accepted_values_orders_status__placed__shipped__completed__return_pending__returned\", \"nativeParameters\": {\"values\": \"['placed', 'shipped', 'completed', 'return_pending', 'returned']\", \"column_name\": \"status\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2647,7 +2655,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_amount\", \"nativeParameters\": {\"column_name\": \"amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_amount\", \"nativeParameters\": {\"column_name\": \"amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2685,7 +2693,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_credit_card_amount\", \"nativeParameters\": {\"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_credit_card_amount\", \"nativeParameters\": {\"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2723,7 +2731,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),credit_card_amount)\"], \"aggregation\": \"_NATIVE_\", \"operator\": \"_NATIVE_\", \"nativeType\": \"dbt_expectations_expect_column_values_to_not_be_in_set_orders_credit_card_amount__credit_card_amount_is_not_null__0\", \"nativeParameters\": {\"value_set\": \"['0']\", \"row_condition\": \"credit_card_amount is not null\", \"column_name\": \"credit_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}, \"logic\": \"\\n\\nwith all_values as (\\n\\n select\\n credit_card_amount as value_field\\n\\n from `calm-pagoda-323403`.`jaffle_shop`.`orders`\\n \\n where credit_card_amount is not null\\n \\n\\n),\\nset_values as (\\n\\n select\\n cast('0' as \\n string\\n) as value_field\\n \\n \\n),\\nvalidation_errors as (\\n -- values from the model that match the set\\n select\\n v.value_field\\n from\\n all_values v\\n join\\n set_values s on v.value_field = s.value_field\\n\\n)\\n\\nselect *\\nfrom validation_errors\\n\\n\"}}", "contentType": "application/json" }, "systemMetadata": { @@ -2761,7 +2769,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),coupon_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_coupon_amount\", \"nativeParameters\": {\"column_name\": \"coupon_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),coupon_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_coupon_amount\", \"nativeParameters\": {\"column_name\": \"coupon_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2799,7 +2807,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),bank_transfer_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_bank_transfer_amount\", \"nativeParameters\": {\"column_name\": \"bank_transfer_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),bank_transfer_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_bank_transfer_amount\", \"nativeParameters\": {\"column_name\": \"bank_transfer_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -2837,7 +2845,7 @@ "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { - "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),gift_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_gift_card_amount\", \"nativeParameters\": {\"column_name\": \"gift_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", + "value": "{\"customProperties\": {\"manifest_schema\": \"https://schemas.getdbt.com/dbt/manifest/v5.json\", \"manifest_version\": \"1.1.0\", \"manifest_adapter\": \"bigquery\", \"catalog_schema\": \"https://schemas.getdbt.com/dbt/catalog/v1.json\", \"catalog_version\": \"1.1.0\"}, \"type\": \"DATASET\", \"datasetAssertion\": {\"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"scope\": \"DATASET_COLUMN\", \"fields\": [\"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD),gift_card_amount)\"], \"aggregation\": \"IDENTITY\", \"operator\": \"NOT_NULL\", \"nativeType\": \"not_null_orders_gift_card_amount\", \"nativeParameters\": {\"column_name\": \"gift_card_amount\", \"model\": \"{{ get_where_subquery(ref('orders')) }}\"}}}", "contentType": "application/json" }, "systemMetadata": { @@ -3304,4 +3312,4 @@ "properties": null } } -] \ No newline at end of file +] From 8379bc7e1ce2f45349a15c32798fb5cc809bab4e Mon Sep 17 00:00:00 2001 From: Ravindra Lanka Date: Wed, 3 Aug 2022 10:44:21 -0700 Subject: [PATCH 10/10] Minor refactoring & interface cleanup. --- .../src/datahub/ingestion/source/dbt.py | 33 +++++++++---------- .../ingestion/source/state/dbt_state.py | 10 +++--- .../source/state/sql_common_state.py | 12 ++----- .../source/state/stateful_ingestion_base.py | 2 +- ...point_util.py => checkpoint_state_util.py} | 1 - .../tests/integration/dbt/test_dbt.py | 17 +++------- .../tests/test_helpers/state_helpers.py | 5 --- 7 files changed, 29 insertions(+), 51 deletions(-) rename metadata-ingestion/src/datahub/utilities/{check_point_util.py => checkpoint_state_util.py} (99%) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index c6e9e9d803b506..3b57f438ec4d63 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -46,7 +46,7 @@ resolve_postgres_modified_type, resolve_trino_modified_type, ) -from datahub.ingestion.source.state.checkpoint import Checkpoint, CheckpointStateBase +from datahub.ingestion.source.state.checkpoint import Checkpoint from datahub.ingestion.source.state.dbt_state import DbtCheckpointState from datahub.ingestion.source.state.sql_common_state import ( BaseSQLAlchemyCheckpointState, @@ -1007,29 +1007,26 @@ def __init__(self, config: DBTConfig, ctx: PipelineContext, platform: str): ) def get_last_dbt_checkpoint( - self, job_id: JobId, checkpoint_state_class: Type[CheckpointStateBase] + self, job_id: JobId, checkpoint_state_class: Type[DbtCheckpointState] ) -> Optional[Checkpoint]: - last_checkpoint: Optional[Checkpoint] = cast(Checkpoint, None) + last_checkpoint: Optional[Checkpoint] is_conversion_required: bool = False try: # Best-case that last checkpoint state is DbtCheckpointState - last_checkpoint = self.get_last_checkpoint( - self.get_default_ingestion_job_id(), DbtCheckpointState - ) - except Exception: + last_checkpoint = self.get_last_checkpoint(job_id, checkpoint_state_class) + except Exception as e: # Backward compatibility for old dbt ingestion source which was saving dbt-nodes in # BaseSQLAlchemyCheckpointState last_checkpoint = self.get_last_checkpoint( - self.get_default_ingestion_job_id(), BaseSQLAlchemyCheckpointState + job_id, BaseSQLAlchemyCheckpointState + ) + logger.debug( + f"Found BaseSQLAlchemyCheckpointState as checkpoint state (got {e})." ) - logger.debug("Found BaseSQLAlchemyCheckpointState as checkpoint state") is_conversion_required = True - if last_checkpoint is None: - return None - - if is_conversion_required: + if last_checkpoint is not None and is_conversion_required: # Map the BaseSQLAlchemyCheckpointState to DbtCheckpointState dbt_checkpoint_state: DbtCheckpointState = DbtCheckpointState() dbt_checkpoint_state.encoded_node_urns = ( @@ -1061,7 +1058,7 @@ def gen_removed_entity_workunits(self) -> Iterable[MetadataWorkUnit]: ): logger.debug("Checking for stale entity removal.") - def soft_delete_item(urn: str, type: str) -> MetadataWorkUnit: + def get_soft_delete_item_workunit(urn: str, type: str) -> MetadataWorkUnit: logger.info(f"Soft-deleting stale entity of type {type} - {urn}.") mcp = MetadataChangeProposalWrapper( @@ -1079,7 +1076,7 @@ def soft_delete_item(urn: str, type: str) -> MetadataWorkUnit: last_checkpoint_state = cast(DbtCheckpointState, last_checkpoint.state) cur_checkpoint_state = cast(DbtCheckpointState, cur_checkpoint.state) - soft_delete_urn: Dict = { + urns_to_soft_delete_by_type: Dict = { "dataset": [ node_urn for node_urn in last_checkpoint_state.get_node_urns_not_in( @@ -1093,9 +1090,9 @@ def soft_delete_item(urn: str, type: str) -> MetadataWorkUnit: ) ], } - for entity_type in soft_delete_urn: - for urn in soft_delete_urn[entity_type]: - yield soft_delete_item(urn, entity_type) + for entity_type in urns_to_soft_delete_by_type: + for urn in urns_to_soft_delete_by_type[entity_type]: + yield get_soft_delete_item_workunit(urn, entity_type) def load_file_as_json(self, uri: str) -> Any: if re.match("^https?://", uri): diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py index 6746619a5de3ce..04c9abb81a165c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py @@ -1,11 +1,11 @@ import logging -from typing import Iterable, List +from typing import Callable, Dict, Iterable, List import pydantic from datahub.emitter.mce_builder import make_assertion_urn from datahub.ingestion.source.state.checkpoint import CheckpointStateBase -from datahub.utilities.check_point_util import CheckpointStateUtil +from datahub.utilities.checkpoint_state_util import CheckpointStateUtil from datahub.utilities.urns.urn import Urn logger = logging.getLogger(__name__) @@ -59,12 +59,12 @@ def add_node_urn(self, node_urn: str) -> None: ) def set_checkpoint_urn(self, urn: str, entity_type: str) -> None: - supported_entities = { + supported_entities_add_handlers: Dict[str, Callable[[str], None]] = { "dataset": self.add_node_urn, "assertion": self.add_assertion_urn, } - if supported_entities.get(entity_type) is None: + if entity_type not in supported_entities_add_handlers: logger.error(f"Can not save Unknown entity {entity_type} to checkpoint.") - supported_entities[entity_type](urn) + supported_entities_add_handlers[entity_type](urn) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py index 65db34c23d4c91..8186232b30ee16 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py @@ -4,7 +4,7 @@ from datahub.emitter.mce_builder import container_urn_to_key, make_container_urn from datahub.ingestion.source.state.checkpoint import CheckpointStateBase -from datahub.utilities.check_point_util import CheckpointStateUtil +from datahub.utilities.checkpoint_state_util import CheckpointStateUtil class BaseSQLAlchemyCheckpointState(CheckpointStateBase): @@ -44,10 +44,7 @@ def _get_container_urns_not_in( def get_table_urns_not_in( self, checkpoint: "BaseSQLAlchemyCheckpointState" ) -> Iterable[str]: - """ - Tables are mapped to DataHub dataset concept - """ - + """Tables are mapped to DataHub dataset concept.""" yield from CheckpointStateUtil.get_dataset_urns_not_in( self.encoded_table_urns, checkpoint.encoded_table_urns ) @@ -55,10 +52,7 @@ def get_table_urns_not_in( def get_view_urns_not_in( self, checkpoint: "BaseSQLAlchemyCheckpointState" ) -> Iterable[str]: - """ - Views are mapped to DataHub dataset concept - """ - + """Views are mapped to DataHub dataset concept.""" yield from CheckpointStateUtil.get_dataset_urns_not_in( self.encoded_view_urns, checkpoint.encoded_view_urns ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py index 066c1676eff918..9ae0aee12fabd3 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py @@ -232,7 +232,7 @@ def get_last_checkpoint( ): return None - if JobId not in self.last_checkpoints: + if job_id not in self.last_checkpoints: self.last_checkpoints[job_id] = self._get_last_checkpoint( job_id, checkpoint_state_class ) diff --git a/metadata-ingestion/src/datahub/utilities/check_point_util.py b/metadata-ingestion/src/datahub/utilities/checkpoint_state_util.py similarity index 99% rename from metadata-ingestion/src/datahub/utilities/check_point_util.py rename to metadata-ingestion/src/datahub/utilities/checkpoint_state_util.py index 4208b859b615c6..d069dedceb9ae9 100644 --- a/metadata-ingestion/src/datahub/utilities/check_point_util.py +++ b/metadata-ingestion/src/datahub/utilities/checkpoint_state_util.py @@ -17,7 +17,6 @@ def get_separator() -> str: def get_encoded_urns_not_in( encoded_urns_1: List[str], encoded_urns_2: List[str] ) -> Set[str]: - return set(encoded_urns_1) - set(encoded_urns_2) @staticmethod diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 4d357e5f402f26..05a6f80c2dddde 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -21,7 +21,6 @@ ) from tests.test_helpers import mce_helpers from tests.test_helpers.state_helpers import ( - create_pipeline, run_and_get_pipeline, validate_all_providers_have_committed_successfully, ) @@ -442,12 +441,11 @@ def test_dbt_state_backward_compatibility( pytestconfig, tmp_path, mock_time, mock_datahub_graph ): test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" + manifest_path = f"{test_resources_dir}/dbt_manifest.json" + catalog_path = f"{test_resources_dir}/dbt_catalog.json" + sources_path = f"{test_resources_dir}/dbt_sources.json" - manifest_path = "{}/dbt_manifest.json".format(test_resources_dir) - catalog_path = "{}/dbt_catalog.json".format(test_resources_dir) - sources_path = "{}/dbt_sources.json".format(test_resources_dir) - - stateful_config = { + stateful_config: Dict[str, Any] = { "stateful_ingestion": { "enabled": True, "remove_stale_metadata": True, @@ -488,9 +486,7 @@ def test_dbt_state_backward_compatibility( mock_datahub_graph, ) as mock_checkpoint: mock_checkpoint.return_value = mock_datahub_graph - - pipeline = create_pipeline(pipeline_config_dict) - + pipeline = Pipeline.create(pipeline_config_dict) dbt_source = cast(DBTSource, pipeline.source) def get_fake_base_sql_alchemy_checkpoint_state( @@ -502,12 +498,10 @@ def get_fake_base_sql_alchemy_checkpoint_state( ) sql_state = BaseSQLAlchemyCheckpointState() - urn1 = "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)" urn2 = ( "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)" ) - sql_state.add_table_urn(urn1) sql_state.add_table_urn(urn2) @@ -524,7 +518,6 @@ def get_fake_base_sql_alchemy_checkpoint_state( # Set fake method to return BaseSQLAlchemyCheckpointState dbt_source.get_last_checkpoint = get_fake_base_sql_alchemy_checkpoint_state # type: ignore[assignment] - last_checkpoint = dbt_source.get_last_dbt_checkpoint( dbt_source.get_default_ingestion_job_id(), DbtCheckpointState ) diff --git a/metadata-ingestion/tests/test_helpers/state_helpers.py b/metadata-ingestion/tests/test_helpers/state_helpers.py index 4da41988a97ac6..4f26b6d87d7ae0 100644 --- a/metadata-ingestion/tests/test_helpers/state_helpers.py +++ b/metadata-ingestion/tests/test_helpers/state_helpers.py @@ -35,11 +35,6 @@ def run_and_get_pipeline(pipeline_config_dict: Dict[str, Any]) -> Pipeline: return pipeline -def create_pipeline(pipeline_config_dict: Dict[str, Any]) -> Pipeline: - pipeline = Pipeline.create(pipeline_config_dict) - return pipeline - - @pytest.fixture def mock_datahub_graph(): class MockDataHubGraphContext: