diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index af4d3545399e16..481e10321d2868 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -29,6 +29,7 @@ ConfigEnum, ConfigModel, ConfigurationError, + LineageConfig, ) from datahub.configuration.github import GitHubReference from datahub.emitter import mce_builder @@ -122,6 +123,7 @@ UpstreamLineageClass, ViewPropertiesClass, ) +from datahub.specific.dataset import DatasetPatchBuilder from datahub.utilities.mapping import Constants, OperationProcessor from datahub.utilities.time import datetime_to_ts_millis @@ -214,7 +216,7 @@ def can_emit_test_results(self) -> bool: return self.test_results == EmitDirective.YES -class DBTConfig(StatefulIngestionConfigBase): +class DBTConfig(StatefulIngestionConfigBase, LineageConfig): manifest_path: str = Field( description="Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json Note this can be a local file or a URI." ) @@ -1447,7 +1449,25 @@ def create_platform_mces( self.config.platform_instance, ) upstreams_lineage_class = get_upstream_lineage([upstream_dbt_urn]) - aspects.append(upstreams_lineage_class) + if self.config.incremental_lineage: + patch_builder: DatasetPatchBuilder = DatasetPatchBuilder( + urn=node_datahub_urn + ) + for upstream in upstreams_lineage_class.upstreams: + patch_builder.add_upstream_lineage(upstream) + + lineage_workunits = [ + MetadataWorkUnit( + id=f"upstreamLineage-for-{node_datahub_urn}", + mcp_raw=mcp, + ) + for mcp in patch_builder.build() + ] + for wu in lineage_workunits: + yield wu + self.report.report_workunit(wu) + else: + aspects.append(upstreams_lineage_class) if len(aspects) == 0: continue diff --git a/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json index 4548bff5696bd8..f9861fb09c8d2a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -12,14 +10,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", @@ -37,11 +31,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payments_by_customer_by_month", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -57,17 +48,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -77,11 +63,7 @@ "fields": [ { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -89,19 +71,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -109,19 +83,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -129,16 +95,9 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -147,14 +106,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -167,20 +124,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -189,14 +140,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", @@ -213,11 +160,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "customer_details", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -233,27 +177,19 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { "tableSchema": "" } }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fields": [] } }, { @@ -262,8 +198,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" @@ -271,8 +206,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" @@ -280,14 +214,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -300,20 +232,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -322,14 +248,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", @@ -347,11 +269,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -367,17 +286,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -387,11 +301,7 @@ "fields": [ { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -399,19 +309,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -419,19 +321,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -439,19 +333,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -459,16 +345,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -477,8 +356,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" @@ -486,14 +364,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -506,20 +382,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -528,14 +398,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", @@ -553,11 +419,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "an-aliased-view-for-payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -573,17 +436,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -593,11 +451,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -605,19 +459,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -625,19 +471,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -645,19 +483,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -665,19 +495,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -685,19 +507,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -705,16 +519,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -723,8 +530,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" @@ -732,8 +538,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" @@ -741,8 +546,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -750,8 +554,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" @@ -759,8 +562,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" @@ -768,14 +570,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -788,20 +588,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -810,14 +604,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", @@ -834,11 +624,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "address", - "qualifiedName": null, "description": "a user's address", - "uri": null, "tags": [] } }, @@ -854,17 +641,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -874,11 +656,7 @@ "fields": [ { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -886,19 +664,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -906,19 +676,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address2", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -926,19 +688,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "district", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -946,19 +700,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -966,19 +712,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "postal_code", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -986,19 +724,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "phone", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1006,19 +736,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1026,16 +748,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1044,33 +759,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1079,14 +786,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", @@ -1103,11 +806,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "category", - "qualifiedName": null, "description": "a user's category", - "uri": null, "tags": [] } }, @@ -1123,17 +823,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1143,11 +838,7 @@ "fields": [ { "fieldPath": "category_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1155,19 +846,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1175,19 +858,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1195,16 +870,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1213,33 +881,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1248,14 +908,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", @@ -1272,11 +928,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "city", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1292,17 +945,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1312,11 +960,7 @@ "fields": [ { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1324,19 +968,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1344,19 +980,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1364,19 +992,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1384,16 +1004,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1402,33 +1015,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1437,14 +1042,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", @@ -1464,11 +1065,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "country", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1482,14 +1080,12 @@ "owners": [ { "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -1500,17 +1096,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1520,11 +1111,7 @@ "fields": [ { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1532,19 +1119,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1552,19 +1131,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1572,16 +1143,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1590,33 +1154,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1625,14 +1181,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", @@ -1649,11 +1201,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "customer", - "qualifiedName": null, "description": "description for customer table from dbt", - "uri": null, "tags": [] } }, @@ -1669,17 +1218,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1689,11 +1233,7 @@ "fields": [ { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1701,19 +1241,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "store_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1721,19 +1253,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1741,19 +1265,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1761,19 +1277,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1781,19 +1289,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1801,19 +1301,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "activebool", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1821,19 +1313,11 @@ }, "nativeDataType": "boolean", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "create_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1841,19 +1325,11 @@ }, "nativeDataType": "date", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1861,19 +1337,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "active", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1881,16 +1349,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1899,33 +1360,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1934,14 +1387,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", @@ -1958,11 +1407,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payment_p2020_01", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1978,17 +1424,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1580505371997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1998,11 +1439,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2010,19 +1447,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2030,19 +1459,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2050,19 +1471,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2070,19 +1483,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2090,19 +1495,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2110,16 +1507,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2128,33 +1518,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2163,14 +1545,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", @@ -2191,11 +1569,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payment_p2020_02", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2209,14 +1584,12 @@ "owners": [ { "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -2227,17 +1600,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1582319845997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2247,11 +1615,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2259,19 +1623,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2279,19 +1635,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2299,19 +1647,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2319,19 +1659,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2339,19 +1671,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2359,16 +1683,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2377,33 +1694,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2412,14 +1721,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", @@ -2436,11 +1741,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payment_p2020_03", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2456,17 +1758,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1584998318997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2476,11 +1773,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2488,19 +1781,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2508,19 +1793,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2528,19 +1805,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2548,19 +1817,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2568,19 +1829,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2588,16 +1841,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2606,33 +1852,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2641,14 +1879,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", @@ -2665,11 +1899,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payment_p2020_04", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2685,17 +1916,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1588287228997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2705,11 +1931,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2717,19 +1939,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2737,19 +1951,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2757,19 +1963,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2777,19 +1975,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2797,19 +1987,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2817,16 +1999,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2835,33 +2010,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2870,14 +2037,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", @@ -2894,11 +2057,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payment_p2020_05", - "qualifiedName": null, "description": "a payment", - "uri": null, "tags": [] } }, @@ -2914,17 +2074,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1589460269997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2934,11 +2089,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2946,19 +2097,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2966,19 +2109,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2986,19 +2121,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3006,19 +2133,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3026,19 +2145,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3046,16 +2157,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3064,33 +2168,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3099,14 +2195,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", @@ -3123,11 +2215,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.0.3" }, - "externalUrl": null, "name": "payment_p2020_06", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -3143,17 +2232,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3163,11 +2247,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3175,19 +2255,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3195,19 +2267,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3215,19 +2279,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3235,19 +2291,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3255,19 +2303,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3275,16 +2315,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3293,135 +2326,67 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -3430,17 +2395,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -3449,10 +2409,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "dbt-2022_02_03-07_00_00", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-2022_02_03-07_00_00" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index 7313495e9fb9c6..5d44912d164ee2 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -12,14 +10,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", @@ -42,11 +36,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer_details", - "qualifiedName": null, "description": "", - "uri": null, "tags": [ "dbt:test_tag" ] @@ -64,23 +55,20 @@ "owner": "urn:li:corpuser:alice2", "type": "DATAOWNER", "source": { - "type": "SOURCE_CONTROL", - "url": null + "type": "SOURCE_CONTROL" } }, { "owner": "urn:li:corpuser:jdoe.last@gmail.com", "type": "DATAOWNER", "source": { - "type": "SOURCE_CONTROL", - "url": null + "type": "SOURCE_CONTROL" } } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -88,20 +76,16 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:has_pii_test", - "context": null + "tag": "urn:li:tag:dbt:has_pii_test" }, { - "tag": "urn:li:tag:dbt:int_meta_property", - "context": null + "tag": "urn:li:tag:dbt:int_meta_property" }, { - "tag": "urn:li:tag:dbt:my_query_tag", - "context": null + "tag": "urn:li:tag:dbt:my_query_tag" }, { - "tag": "urn:li:tag:dbt:test_tag", - "context": null + "tag": "urn:li:tag:dbt:test_tag" } ] } @@ -110,18 +94,15 @@ "com.linkedin.pegasus2avro.common.GlossaryTerms": { "terms": [ { - "urn": "urn:li:glossaryTerm:Finance_test", - "context": null + "urn": "urn:li:glossaryTerm:Finance_test" }, { - "urn": "urn:li:glossaryTerm:double_meta_property", - "context": null + "urn": "urn:li:glossaryTerm:double_meta_property" } ], "auditStamp": { "time": 1615443388097, - "actor": "urn:li:corpuser:datahub", - "impersonator": null + "actor": "urn:li:corpuser:datahub" } } }, @@ -132,27 +113,19 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { "tableSchema": "" } }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fields": [] } }, { @@ -161,8 +134,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" @@ -170,8 +142,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" @@ -179,14 +150,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -199,20 +168,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -221,14 +184,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", @@ -251,11 +210,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -271,15 +227,13 @@ "owner": "urn:li:corpuser:jdoe.last", "type": "DATAOWNER", "source": { - "type": "SOURCE_CONTROL", - "url": null + "type": "SOURCE_CONTROL" } } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -290,17 +244,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -310,11 +259,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -322,19 +267,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -342,19 +279,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -362,19 +291,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -382,16 +303,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -400,8 +314,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" @@ -409,14 +322,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -429,20 +340,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -451,14 +356,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", @@ -476,11 +377,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -496,17 +394,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -516,11 +409,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -528,19 +417,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -548,19 +429,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -568,19 +441,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -588,19 +453,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -608,19 +465,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -628,16 +477,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -646,8 +488,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" @@ -655,8 +496,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -664,8 +504,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -673,8 +512,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" @@ -682,8 +520,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" @@ -691,8 +528,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" @@ -700,14 +536,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -720,20 +554,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -742,14 +570,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", @@ -767,11 +591,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payments_by_customer_by_month", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -787,17 +608,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -807,11 +623,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -819,19 +631,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -839,19 +643,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -859,16 +655,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -877,14 +666,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -897,20 +684,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -919,14 +700,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", @@ -946,11 +723,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "actor", - "qualifiedName": null, "description": "description for actor table from dbt", - "uri": null, "tags": [] } }, @@ -966,15 +740,13 @@ "owner": "urn:li:corpuser:alice1", "type": "DATAOWNER", "source": { - "type": "SOURCE_CONTROL", - "url": null + "type": "SOURCE_CONTROL" } } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -985,17 +757,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1005,11 +772,7 @@ "fields": [ { "fieldPath": "actor_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1017,19 +780,12 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1040,23 +796,16 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag", - "context": null + "tag": "urn:li:tag:dbt:column_tag" } ] }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1064,19 +813,12 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1084,16 +826,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1102,33 +837,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1137,14 +864,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", @@ -1161,11 +884,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "address", - "qualifiedName": null, "description": "a user's address", - "uri": null, "tags": [] } }, @@ -1181,17 +901,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1201,11 +916,7 @@ "fields": [ { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1213,19 +924,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address2", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1233,19 +936,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1253,19 +948,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1273,19 +960,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "district", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1293,19 +972,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1313,19 +984,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "phone", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1333,19 +996,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "postal_code", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1353,16 +1008,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1371,33 +1019,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1406,14 +1046,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", @@ -1430,11 +1066,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "category", - "qualifiedName": null, "description": "a user's category", - "uri": null, "tags": [] } }, @@ -1450,17 +1083,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1470,11 +1098,7 @@ "fields": [ { "fieldPath": "category_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1482,19 +1106,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1502,19 +1118,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1522,16 +1130,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1540,33 +1141,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1575,14 +1168,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", @@ -1599,11 +1188,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "city", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1619,17 +1205,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1639,11 +1220,7 @@ "fields": [ { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1651,19 +1228,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1671,19 +1240,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1691,19 +1252,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1711,16 +1264,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1729,33 +1275,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1764,14 +1302,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", @@ -1791,11 +1325,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "country", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1811,15 +1342,13 @@ "owner": "urn:li:corpuser:bob", "type": "DATAOWNER", "source": { - "type": "SOURCE_CONTROL", - "url": null + "type": "SOURCE_CONTROL" } } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -1830,17 +1359,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1850,11 +1374,7 @@ "fields": [ { "fieldPath": "country", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1862,19 +1382,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1882,19 +1394,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1902,16 +1406,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1920,33 +1417,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1955,14 +1444,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", @@ -1979,11 +1464,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer", - "qualifiedName": null, "description": "description for customer table from dbt", - "uri": null, "tags": [] } }, @@ -1999,17 +1481,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2019,11 +1496,7 @@ "fields": [ { "fieldPath": "active", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2031,19 +1504,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "activebool", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -2051,19 +1516,11 @@ }, "nativeDataType": "boolean", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2071,19 +1528,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "create_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -2091,19 +1540,11 @@ }, "nativeDataType": "date", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2111,19 +1552,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2131,19 +1564,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2151,19 +1576,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2171,19 +1588,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2191,19 +1600,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "store_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2211,16 +1612,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2229,33 +1623,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2264,14 +1650,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", @@ -2288,11 +1670,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_01", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2308,17 +1687,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1580505371997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2328,11 +1702,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2340,19 +1710,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2360,19 +1722,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2380,19 +1734,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2400,19 +1746,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2420,19 +1758,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2440,16 +1770,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2458,33 +1781,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2493,14 +1808,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", @@ -2521,11 +1832,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_02", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2541,15 +1849,13 @@ "owner": "urn:li:corpuser:charles", "type": "DATAOWNER", "source": { - "type": "SOURCE_CONTROL", - "url": null + "type": "SOURCE_CONTROL" } } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -2560,17 +1866,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1582319845997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2580,11 +1881,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2592,19 +1889,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2612,19 +1901,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2632,19 +1913,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2652,19 +1925,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2672,19 +1937,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2692,16 +1949,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2710,33 +1960,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2745,14 +1987,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", @@ -2769,11 +2007,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_03", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2789,17 +2024,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1584998318997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2809,11 +2039,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2821,19 +2047,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2841,19 +2059,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2861,19 +2071,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2881,19 +2083,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2901,19 +2095,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2921,16 +2107,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2939,33 +2118,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2974,14 +2145,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", @@ -2998,11 +2165,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_04", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -3018,17 +2182,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1588287228997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3038,11 +2197,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3050,19 +2205,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3070,19 +2217,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3090,19 +2229,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3110,19 +2241,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3130,19 +2253,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3150,16 +2265,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3168,33 +2276,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3203,14 +2303,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", @@ -3227,11 +2323,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_05", - "qualifiedName": null, "description": "a payment", - "uri": null, "tags": [] } }, @@ -3247,17 +2340,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1589460269997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3267,11 +2355,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3279,19 +2363,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3299,19 +2375,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3319,19 +2387,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3339,19 +2399,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3359,19 +2411,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3379,16 +2423,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3397,33 +2434,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3432,14 +2461,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", @@ -3456,11 +2481,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_06", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -3476,17 +2498,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3496,11 +2513,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3508,19 +2521,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3528,19 +2533,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3548,19 +2545,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3568,19 +2557,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3588,19 +2569,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3608,16 +2581,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3626,128 +2592,62 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-schemas-dbt-enabled" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json index eee040cc407a92..c3f679fccc0637 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -12,14 +10,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", @@ -37,11 +31,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "stg_customers", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -57,17 +48,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -77,11 +63,7 @@ "fields": [ { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -89,19 +71,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -109,19 +83,11 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -129,16 +95,9 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -147,14 +106,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -167,20 +124,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -189,14 +140,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", @@ -214,11 +161,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "stg_payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -234,17 +178,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -254,11 +193,7 @@ "fields": [ { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -266,19 +201,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "order_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -286,19 +213,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_method", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -306,19 +225,11 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -326,16 +237,9 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -344,14 +248,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -364,20 +266,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -386,14 +282,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", @@ -411,11 +303,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "stg_orders", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -431,17 +320,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -451,11 +335,7 @@ "fields": [ { "fieldPath": "order_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -463,19 +343,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -483,19 +355,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "order_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -503,19 +367,11 @@ }, "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "status", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -523,16 +379,9 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -541,14 +390,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -561,20 +408,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -583,14 +424,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", @@ -608,11 +445,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "raw_customers", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -628,17 +462,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -648,11 +477,7 @@ "fields": [ { "fieldPath": "id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -660,19 +485,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -680,19 +497,11 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -700,35 +509,22 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -737,14 +533,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", @@ -762,11 +554,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "raw_orders", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -782,17 +571,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -802,11 +586,7 @@ "fields": [ { "fieldPath": "id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -814,19 +594,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "user_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -834,19 +606,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "order_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -854,19 +618,11 @@ }, "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "status", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -874,35 +630,22 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -911,14 +654,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", @@ -936,11 +675,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "raw_payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -956,17 +692,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -976,11 +707,7 @@ "fields": [ { "fieldPath": "id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -988,19 +715,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "order_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1008,19 +727,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_method", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1028,19 +739,11 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1048,35 +751,22 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1085,14 +775,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", @@ -1110,11 +796,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "customers", - "qualifiedName": null, "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", - "uri": null, "tags": [] } }, @@ -1130,17 +813,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1150,11 +828,8 @@ "fields": [ { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, "description": "This is a unique identifier for a customer", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1162,19 +837,12 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, "description": "Customer's first name. PII.", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1182,19 +850,12 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, "description": "Customer's last name. PII.", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1202,19 +863,12 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_order", - "jsonPath": null, "nullable": false, "description": "Date (UTC) of a customer's first order", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1222,19 +876,12 @@ }, "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "most_recent_order", - "jsonPath": null, "nullable": false, "description": "Date (UTC) of a customer's most recent order", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1242,19 +889,12 @@ }, "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "number_of_orders", - "jsonPath": null, "nullable": false, "description": "Count of the number of orders a customer has placed", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1262,19 +902,11 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_lifetime_value", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1282,16 +914,9 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1300,8 +925,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", "type": "TRANSFORMED" @@ -1309,8 +933,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", "type": "TRANSFORMED" @@ -1318,14 +941,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -1338,20 +959,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1360,14 +975,10 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", @@ -1385,11 +996,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "1.1.0" }, - "externalUrl": null, "name": "orders", - "qualifiedName": null, "description": "This table has basic information about orders, as well as some derived facts based on payments", - "uri": null, "tags": [] } }, @@ -1405,17 +1013,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1425,11 +1028,8 @@ "fields": [ { "fieldPath": "order_id", - "jsonPath": null, "nullable": false, "description": "This is a unique identifier for an order", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1437,19 +1037,12 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, "description": "Foreign key to the customers table", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1457,19 +1050,12 @@ }, "nativeDataType": "INT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "order_date", - "jsonPath": null, "nullable": false, "description": "Date (UTC) that the order was placed", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1477,19 +1063,12 @@ }, "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "status", - "jsonPath": null, "nullable": false, "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1497,19 +1076,12 @@ }, "nativeDataType": "STRING", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "credit_card_amount", - "jsonPath": null, "nullable": false, "description": "Amount of the order (AUD) paid for by credit card", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1517,19 +1089,12 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "coupon_amount", - "jsonPath": null, "nullable": false, "description": "Amount of the order (AUD) paid for by coupon", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1537,19 +1102,12 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "bank_transfer_amount", - "jsonPath": null, "nullable": false, "description": "Amount of the order (AUD) paid for by bank transfer", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1557,19 +1115,12 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "gift_card_amount", - "jsonPath": null, "nullable": false, "description": "Amount of the order (AUD) paid for by gift card", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1577,19 +1128,12 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "amount", - "jsonPath": null, "nullable": false, "description": "Total amount (AUD) of the order", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1597,16 +1141,9 @@ }, "nativeDataType": "FLOAT64", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1615,8 +1152,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", "type": "TRANSFORMED" @@ -1624,14 +1160,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -1644,292 +1178,126 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -1938,17 +1306,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -1957,17 +1320,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -1976,17 +1334,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -1995,17 +1348,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2014,17 +1362,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2033,17 +1376,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2052,17 +1390,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2071,17 +1404,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2090,17 +1418,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2109,17 +1432,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2128,17 +1446,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2147,17 +1460,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2166,17 +1474,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2185,17 +1488,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2204,17 +1502,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2223,17 +1516,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2242,17 +1530,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2261,17 +1544,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2280,17 +1558,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2299,17 +1572,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2318,17 +1586,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2337,17 +1600,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2356,17 +1614,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2ff754df689ea951ed2e12cbe356708f", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2375,17 +1628,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2394,17 +1642,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2413,17 +1656,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2432,17 +1670,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2451,17 +1684,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2470,17 +1698,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2489,17 +1712,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2508,17 +1726,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2527,17 +1740,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2546,17 +1754,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2565,17 +1768,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2584,17 +1782,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2603,17 +1796,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2622,17 +1810,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2641,17 +1824,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2660,17 +1838,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2679,17 +1852,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2698,17 +1866,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2717,17 +1880,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2736,17 +1894,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2755,17 +1908,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2774,17 +1922,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2793,17 +1936,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2812,17 +1950,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -2831,17 +1964,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionInfo", "aspect": { @@ -2850,17 +1978,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:da743330013b7e3e3707ac6e526ab408", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2869,17 +1992,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:591d8dc8939e0cf9bf0fd03264ad1a0e", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2888,17 +2006,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2887b9c826e0be6296a37833bdc380bd", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2907,17 +2020,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b052a324c05327985f3b579a19ad7579", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2926,17 +2034,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:bf7fd2b46d2c32ee9bb036acd1559782", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2945,17 +2048,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:44519aa345bf3ea896179f9f352ae946", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2964,17 +2062,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:1c217b7587a0cad47a07a09bfe154055", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -2983,17 +2076,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:bbd78a070092f54313153abec49f6f31", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3002,17 +2090,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:ca065a99637630468f688717590beeab", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3021,17 +2104,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:52d06197762e3608d94609e96f03a0a7", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3040,17 +2118,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:7a305acc5fc049dc9bbd141b814461d0", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3059,17 +2132,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:11087a3d7ae178df22c42922ac8ef8ad", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3078,17 +2146,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b301bb47cc4ebce4e78a194b3de11f25", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3097,17 +2160,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:2e9117138dcc9facda66f1efd55a8cd7", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3116,17 +2174,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:25ebf4faa9b1654ef54c46d975ca0a81", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3135,17 +2188,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b03abcc447aac70bbebb22a8a9d7dbbe", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3154,17 +2202,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:c1eebc71f36690e4523adca30314e927", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3173,17 +2216,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3192,17 +2230,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:b210dbd31c2ee4efc0c24a9e4cf125ef", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3211,17 +2244,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:c51ca9c4b5a1f964bef748f0b8968e71", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3230,17 +2258,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:caa9b8060e214cecab88a92dc39c2e60", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3249,17 +2272,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:54bac90e6785bdefd8685ebf8814c429", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3268,17 +2286,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:815963e1332b46a203504ba46ebfab24", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3287,17 +2300,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } }, { - "auditHeader": null, "entityType": "assertion", "entityUrn": "urn:li:assertion:fac27f352406b941125292413afa8096", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "assertionRunEvent", "aspect": { @@ -3306,10 +2314,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "test_pipeline", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "test_pipeline" } } -] +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json index a271d896c08658..75d1ea08783117 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json @@ -2531,26 +2531,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1615443388097, @@ -2558,26 +2545,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1615443388097, @@ -2585,26 +2559,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1615443388097, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json index e80fb0a142d2cc..4fedcaa131c8c7 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json @@ -1184,26 +1184,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1211,26 +1198,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1238,26 +1212,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1265,26 +1226,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1292,26 +1240,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1319,26 +1254,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1346,26 +1268,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.customers,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, @@ -1373,26 +1282,13 @@ } }, { - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)", - "type": "TRANSFORMED" - } - ] - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.orders,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, "systemMetadata": { "lastObserved": 1643871600000, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index ab19354491f99d..5205b80af17203 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -12,14 +10,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", @@ -41,11 +35,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer_details", - "qualifiedName": null, "description": "", - "uri": null, "tags": [ "dbt:test_tag" ] @@ -61,14 +52,12 @@ "owners": [ { "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -76,8 +65,7 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag", - "context": null + "tag": "urn:li:tag:dbt:test_tag" } ] } @@ -89,27 +77,19 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { "tableSchema": "" } }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fields": [] } }, { @@ -118,8 +98,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" @@ -127,8 +106,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" @@ -136,14 +114,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -156,20 +132,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -178,14 +148,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", @@ -208,11 +174,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -228,17 +191,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -248,11 +206,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -260,19 +214,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -280,19 +226,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -300,19 +238,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -320,16 +250,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -338,8 +261,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" @@ -347,14 +269,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -367,20 +287,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -389,14 +303,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", @@ -414,11 +324,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -434,17 +341,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -454,11 +356,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -466,19 +364,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -486,19 +376,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -506,19 +388,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -526,19 +400,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -546,19 +412,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -566,16 +424,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -584,8 +435,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" @@ -593,8 +443,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -602,8 +451,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -611,8 +459,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" @@ -620,8 +467,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" @@ -629,8 +475,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" @@ -638,14 +483,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -658,20 +501,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -680,14 +517,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", @@ -705,11 +538,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payments_by_customer_by_month", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -725,17 +555,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -745,11 +570,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -757,19 +578,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -777,19 +590,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -797,16 +602,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -815,14 +613,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -835,20 +631,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -857,14 +647,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", @@ -884,11 +670,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "actor", - "qualifiedName": null, "description": "description for actor table from dbt", - "uri": null, "tags": [] } }, @@ -902,14 +685,12 @@ "owners": [ { "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -920,17 +701,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -940,11 +716,7 @@ "fields": [ { "fieldPath": "actor_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -952,19 +724,12 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -975,23 +740,16 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag", - "context": null + "tag": "urn:li:tag:dbt:column_tag" } ] }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -999,19 +757,12 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1019,16 +770,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1037,33 +781,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1072,14 +808,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", @@ -1096,11 +828,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "address", - "qualifiedName": null, "description": "a user's address", - "uri": null, "tags": [] } }, @@ -1116,17 +845,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1136,11 +860,7 @@ "fields": [ { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1148,19 +868,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address2", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1168,19 +880,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1188,19 +892,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1208,19 +904,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "district", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1228,19 +916,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1248,19 +928,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "phone", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1268,19 +940,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "postal_code", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1288,16 +952,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1306,33 +963,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1341,14 +990,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", @@ -1365,11 +1010,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "category", - "qualifiedName": null, "description": "a user's category", - "uri": null, "tags": [] } }, @@ -1385,17 +1027,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1405,11 +1042,7 @@ "fields": [ { "fieldPath": "category_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1417,19 +1050,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1437,19 +1062,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1457,16 +1074,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1475,33 +1085,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1510,14 +1112,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", @@ -1534,11 +1132,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "city", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1554,17 +1149,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1574,11 +1164,7 @@ "fields": [ { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1586,19 +1172,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1606,19 +1184,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1626,19 +1196,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1646,16 +1208,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1664,33 +1219,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1699,14 +1246,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", @@ -1726,11 +1269,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "country", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1744,14 +1284,12 @@ "owners": [ { "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -1762,17 +1300,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1782,11 +1315,7 @@ "fields": [ { "fieldPath": "country", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1794,19 +1323,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1814,19 +1335,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1834,16 +1347,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1852,33 +1358,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1887,14 +1385,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", @@ -1911,11 +1405,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer", - "qualifiedName": null, "description": "description for customer table from dbt", - "uri": null, "tags": [] } }, @@ -1931,17 +1422,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1951,11 +1437,7 @@ "fields": [ { "fieldPath": "active", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1963,19 +1445,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "activebool", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1983,19 +1457,11 @@ }, "nativeDataType": "boolean", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2003,19 +1469,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "create_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -2023,19 +1481,11 @@ }, "nativeDataType": "date", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2043,19 +1493,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2063,19 +1505,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2083,19 +1517,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2103,19 +1529,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2123,19 +1541,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "store_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2143,16 +1553,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2161,33 +1564,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2196,14 +1591,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", @@ -2220,11 +1611,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_01", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2240,17 +1628,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1580505371997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2260,11 +1643,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2272,19 +1651,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2292,19 +1663,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2312,19 +1675,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2332,19 +1687,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2352,19 +1699,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2372,16 +1711,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2390,33 +1722,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2425,14 +1749,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", @@ -2453,11 +1773,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_02", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2471,14 +1788,12 @@ "owners": [ { "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -2489,17 +1804,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1582319845997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2509,11 +1819,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2521,19 +1827,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2541,19 +1839,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2561,19 +1851,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2581,19 +1863,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2601,19 +1875,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2621,16 +1887,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2639,33 +1898,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2674,14 +1925,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", @@ -2698,11 +1945,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_03", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2718,17 +1962,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1584998318997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2738,11 +1977,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2750,19 +1985,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2770,19 +1997,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2790,19 +2009,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2810,19 +2021,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2830,19 +2033,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2850,16 +2045,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2868,33 +2056,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2903,14 +2083,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", @@ -2927,11 +2103,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_04", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2947,17 +2120,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1588287228997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2967,11 +2135,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2979,19 +2143,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2999,19 +2155,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3019,19 +2167,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3039,19 +2179,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3059,19 +2191,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3079,16 +2203,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3097,33 +2214,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3132,14 +2241,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", @@ -3156,11 +2261,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_05", - "qualifiedName": null, "description": "a payment", - "uri": null, "tags": [] } }, @@ -3176,17 +2278,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1589460269997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3196,11 +2293,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3208,19 +2301,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3228,19 +2313,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3248,19 +2325,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3268,19 +2337,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3288,19 +2349,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3308,16 +2361,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3326,128 +2372,62 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-complex-owner-patterns", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-complex-owner-patterns" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index 198e433113045b..6935fca22c1253 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -12,14 +10,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)", @@ -42,11 +36,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer_details", - "qualifiedName": null, "description": "", - "uri": null, "tags": [ "dbt:test_tag" ] @@ -62,14 +53,12 @@ "owners": [ { "owner": "urn:li:corpuser:@alice2", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -77,8 +66,7 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag", - "context": null + "tag": "urn:li:tag:dbt:test_tag" } ] } @@ -90,27 +78,19 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { "tableSchema": "" } }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fields": [] } }, { @@ -119,8 +99,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", "type": "TRANSFORMED" @@ -128,8 +107,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", "type": "TRANSFORMED" @@ -137,14 +115,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -157,20 +133,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -179,14 +149,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", @@ -209,11 +175,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -229,17 +192,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -249,11 +207,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -261,19 +215,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -281,19 +227,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -301,19 +239,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -321,16 +251,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -339,8 +262,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" @@ -348,14 +270,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -368,20 +288,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -390,14 +304,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", @@ -415,11 +325,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -435,17 +342,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -455,11 +357,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -467,19 +365,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -487,19 +377,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -507,19 +389,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -527,19 +401,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -547,19 +413,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -567,16 +425,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -585,8 +436,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" @@ -594,8 +444,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -603,8 +452,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -612,8 +460,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" @@ -621,8 +468,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" @@ -630,8 +476,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" @@ -639,14 +484,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -659,20 +502,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -681,14 +518,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", @@ -706,11 +539,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payments_by_customer_by_month", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -726,17 +556,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -746,11 +571,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -758,19 +579,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -778,19 +591,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -798,16 +603,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -816,14 +614,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -836,20 +632,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.actor,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -858,14 +648,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.actor,PROD)", @@ -885,11 +671,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "actor", - "qualifiedName": null, "description": "description for actor table from dbt", - "uri": null, "tags": [] } }, @@ -903,14 +686,12 @@ "owners": [ { "owner": "urn:li:corpuser:@alice1", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -921,17 +702,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -941,11 +717,7 @@ "fields": [ { "fieldPath": "actor_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -953,19 +725,12 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -976,23 +741,16 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag", - "context": null + "tag": "urn:li:tag:dbt:column_tag" } ] }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1000,19 +758,12 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1020,16 +771,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1038,33 +782,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1073,14 +809,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", @@ -1097,11 +829,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "address", - "qualifiedName": null, "description": "a user's address", - "uri": null, "tags": [] } }, @@ -1117,17 +846,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1137,11 +861,7 @@ "fields": [ { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1149,19 +869,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address2", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1169,19 +881,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1189,19 +893,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1209,19 +905,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "district", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1229,19 +917,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1249,19 +929,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "phone", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1269,19 +941,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "postal_code", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1289,16 +953,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1307,33 +964,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.category,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1342,14 +991,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.category,PROD)", @@ -1366,11 +1011,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "category", - "qualifiedName": null, "description": "a user's category", - "uri": null, "tags": [] } }, @@ -1386,17 +1028,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1406,11 +1043,7 @@ "fields": [ { "fieldPath": "category_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1418,19 +1051,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1438,19 +1063,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1458,16 +1075,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1476,33 +1086,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1511,14 +1113,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", @@ -1535,11 +1133,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "city", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1555,17 +1150,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1575,11 +1165,7 @@ "fields": [ { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1587,19 +1173,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1607,19 +1185,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1627,19 +1197,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1647,16 +1209,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1665,33 +1220,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.country,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1700,14 +1247,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.country,PROD)", @@ -1727,11 +1270,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "country", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1745,14 +1285,12 @@ "owners": [ { "owner": "urn:li:corpuser:@bob", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -1763,17 +1301,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1783,11 +1316,7 @@ "fields": [ { "fieldPath": "country", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1795,19 +1324,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1815,19 +1336,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1835,16 +1348,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1853,33 +1359,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1888,14 +1386,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", @@ -1912,11 +1406,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer", - "qualifiedName": null, "description": "description for customer table from dbt", - "uri": null, "tags": [] } }, @@ -1932,17 +1423,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1952,11 +1438,7 @@ "fields": [ { "fieldPath": "active", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1964,19 +1446,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "activebool", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1984,19 +1458,11 @@ }, "nativeDataType": "boolean", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2004,19 +1470,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "create_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -2024,19 +1482,11 @@ }, "nativeDataType": "date", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2044,19 +1494,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2064,19 +1506,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2084,19 +1518,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2104,19 +1530,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2124,19 +1542,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "store_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2144,16 +1554,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2162,33 +1565,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2197,14 +1592,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_01,PROD)", @@ -2221,11 +1612,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_01", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2241,17 +1629,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1580505371997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2261,11 +1644,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2273,19 +1652,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2293,19 +1664,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2313,19 +1676,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2333,19 +1688,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2353,19 +1700,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2373,16 +1712,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2391,33 +1723,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2426,14 +1750,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", @@ -2454,11 +1774,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_02", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2472,14 +1789,12 @@ "owners": [ { "owner": "urn:li:corpuser:@charles", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -2490,17 +1805,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1582319845997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2510,11 +1820,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2522,19 +1828,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2542,19 +1840,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2562,19 +1852,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,19 +1864,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2602,19 +1876,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2622,16 +1888,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2640,33 +1899,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2675,14 +1926,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_03,PROD)", @@ -2699,11 +1946,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_03", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2719,17 +1963,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1584998318997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2739,11 +1978,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2751,19 +1986,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2771,19 +1998,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2791,19 +2010,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2811,19 +2022,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2831,19 +2034,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2851,16 +2046,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2869,33 +2057,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2904,14 +2084,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_04,PROD)", @@ -2928,11 +2104,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_04", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2948,17 +2121,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1588287228997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2968,11 +2136,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2980,19 +2144,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3000,19 +2156,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3020,19 +2168,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3040,19 +2180,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3060,19 +2192,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3080,16 +2204,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3098,33 +2215,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3133,14 +2242,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD)", @@ -3157,11 +2262,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_05", - "qualifiedName": null, "description": "a payment", - "uri": null, "tags": [] } }, @@ -3177,17 +2279,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1589460269997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3197,11 +2294,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3209,19 +2302,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3229,19 +2314,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3249,19 +2326,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3269,19 +2338,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3289,19 +2350,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3309,16 +2362,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3327,33 +2373,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3362,14 +2400,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD)", @@ -3386,11 +2420,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_06", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -3406,17 +2437,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3426,11 +2452,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3438,19 +2460,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3458,19 +2472,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3478,19 +2484,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3498,19 +2496,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3518,19 +2508,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3538,16 +2520,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3556,128 +2531,62 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-data-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-data-platform-instance" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json new file mode 100644 index 00000000000000..74d08ab89b65c6 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json @@ -0,0 +1,2631 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "owner": "@alice2", + "business_owner": "jdoe.last@gmail.com", + "data_governance.team_owner": "Finance", + "has_pii": "True", + "int_property": "1", + "double_property": "2.5", + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transform/customer_details.sql", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "customer_details", + "description": "", + "tags": [ + "dbt:test_tag" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice2", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:test_tag" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.customer_details", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "business_owner": "jdoe.last", + "data_governance.team_owner": "Sales", + "has_pii": "False", + "int_property": "2", + "double_property": "3.5", + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "an-aliased-view-for-monthly-billing", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.monthly_billing_with_cust", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billing_month", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/base/payments_base.sql", + "catalog_type": "VIEW", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "an-aliased-view-for-payments", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_base", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized=\"view\",\n alias='an-aliased-view-for-payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\", \"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payments_by_customer_by_month", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_by_customer_by_month", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billing_month", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n billing_month,\n customer_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in dev", + "some_other_property": "test 1", + "owner": "@alice1", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "actor", + "description": "description for actor table from dbt", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice1", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.actor", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759273000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:column_tag" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "description": "description for last_name from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "description": "description for last_update from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "address", + "description": "a user's address", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.address", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759930000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "category", + "description": "a user's category", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.category", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759987000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "category_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "city", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.city", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759925000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "country_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "country", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@bob", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.country", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759840000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "country", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "country_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "customer", + "description": "description for customer table from dbt", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.customer", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581760640000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activebool", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "create_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payment_p2020_01", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_01", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1580505371997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "an_array_property": "['alpha', 'beta', 'charlie']", + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payment_p2020_02", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@charles", + "type": "DATAOWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_02", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1582319845997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payment_p2020_03", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_03", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1584998318997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payment_p2020_04", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_04", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1588287228997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payment_p2020_05", + "description": "a payment", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_05", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1589460269997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"source\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "manifest_version": "0.19.1", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "0.19.1" + }, + "name": "payment_p2020_06", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_06", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": -62135596800000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "dbt-test-with-non-incremental-lineage" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index 5779e077c3272d..294bf22dbf68e1 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -12,14 +10,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", @@ -42,11 +36,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer_details", - "qualifiedName": null, "description": "", - "uri": null, "tags": [ "dbt:test_tag" ] @@ -62,14 +53,12 @@ "owners": [ { "owner": "urn:li:corpuser:@alice2", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -77,8 +66,7 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag", - "context": null + "tag": "urn:li:tag:dbt:test_tag" } ] } @@ -90,27 +78,19 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { "tableSchema": "" } }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fields": [] } }, { @@ -119,8 +99,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" @@ -128,8 +107,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" @@ -137,14 +115,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -157,20 +133,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -179,14 +149,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", @@ -209,11 +175,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -229,17 +192,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -249,11 +207,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -261,19 +215,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -281,19 +227,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -301,19 +239,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -321,16 +251,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -339,8 +262,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "type": "TRANSFORMED" @@ -348,14 +270,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -368,20 +288,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -390,14 +304,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", @@ -415,11 +325,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "an-aliased-view-for-payments", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -435,17 +342,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -455,11 +357,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -467,19 +365,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -487,19 +377,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -507,19 +389,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -527,19 +401,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -547,19 +413,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -567,16 +425,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -585,8 +436,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" @@ -594,8 +444,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -603,8 +452,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" @@ -612,8 +460,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" @@ -621,8 +468,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" @@ -630,8 +476,7 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" @@ -639,14 +484,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -659,20 +502,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -681,14 +518,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", @@ -706,11 +539,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payments_by_customer_by_month", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -726,17 +556,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -746,11 +571,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -758,19 +579,11 @@ }, "nativeDataType": "numeric", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "billing_month", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -778,19 +591,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -798,16 +603,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -816,14 +614,12 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } }, { @@ -836,20 +632,14 @@ ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -858,14 +648,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", @@ -885,11 +671,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "actor", - "qualifiedName": null, "description": "description for actor table from dbt", - "uri": null, "tags": [] } }, @@ -903,14 +686,12 @@ "owners": [ { "owner": "urn:li:corpuser:@alice1", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -921,17 +702,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -941,11 +717,7 @@ "fields": [ { "fieldPath": "actor_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -953,19 +725,12 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -976,23 +741,16 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag", - "context": null + "tag": "urn:li:tag:dbt:column_tag" } ] }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1000,19 +758,12 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1020,16 +771,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1038,33 +782,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.actor,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1073,14 +809,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", @@ -1097,11 +829,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "address", - "qualifiedName": null, "description": "a user's address", - "uri": null, "tags": [] } }, @@ -1117,17 +846,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1137,11 +861,7 @@ "fields": [ { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1149,19 +869,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address2", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1169,19 +881,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1189,19 +893,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1209,19 +905,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "district", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1229,19 +917,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1249,19 +929,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "phone", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1269,19 +941,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "postal_code", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1289,16 +953,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1307,33 +964,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.address,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1342,14 +991,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", @@ -1366,11 +1011,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "category", - "qualifiedName": null, "description": "a user's category", - "uri": null, "tags": [] } }, @@ -1386,17 +1028,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1406,11 +1043,7 @@ "fields": [ { "fieldPath": "category_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1418,19 +1051,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1438,19 +1063,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1458,16 +1075,9 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1476,33 +1086,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.category,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1511,14 +1113,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", @@ -1535,11 +1133,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "city", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1555,17 +1150,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1575,11 +1165,7 @@ "fields": [ { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1587,19 +1173,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1607,19 +1185,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1627,19 +1197,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1647,16 +1209,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1665,33 +1220,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.city,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1700,14 +1247,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", @@ -1727,11 +1270,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "country", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -1745,14 +1285,12 @@ "owners": [ { "owner": "urn:li:corpuser:@bob", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -1763,17 +1301,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1783,11 +1316,7 @@ "fields": [ { "fieldPath": "country", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1795,19 +1324,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "country_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1815,19 +1336,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1835,16 +1348,9 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -1853,33 +1359,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.country,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1888,14 +1386,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", @@ -1912,11 +1406,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "customer", - "qualifiedName": null, "description": "description for customer table from dbt", - "uri": null, "tags": [] } }, @@ -1932,17 +1423,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -1952,11 +1438,7 @@ "fields": [ { "fieldPath": "active", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1964,19 +1446,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "activebool", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1984,19 +1458,11 @@ }, "nativeDataType": "boolean", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2004,19 +1470,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "create_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -2024,19 +1482,11 @@ }, "nativeDataType": "date", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2044,19 +1494,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "email", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2064,19 +1506,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "first_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2084,19 +1518,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_name", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -2104,19 +1530,11 @@ }, "nativeDataType": "text", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "last_update", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2124,19 +1542,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "store_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2144,16 +1554,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2162,33 +1565,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.customer,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2197,14 +1592,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", @@ -2221,11 +1612,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_01", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2241,17 +1629,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1580505371997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2261,11 +1644,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2273,19 +1652,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2293,19 +1664,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2313,19 +1676,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2333,19 +1688,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2353,19 +1700,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2373,16 +1712,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2391,33 +1723,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2426,14 +1750,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", @@ -2454,11 +1774,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_02", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2472,14 +1789,12 @@ "owners": [ { "owner": "urn:li:corpuser:@charles", - "type": "DATAOWNER", - "source": null + "type": "DATAOWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -2490,17 +1805,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1582319845997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2510,11 +1820,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2522,19 +1828,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2542,19 +1840,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2562,19 +1852,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,19 +1864,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2602,19 +1876,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2622,16 +1888,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2640,33 +1899,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2675,14 +1926,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", @@ -2699,11 +1946,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_03", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2719,17 +1963,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1584998318997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2739,11 +1978,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2751,19 +1986,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2771,19 +1998,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2791,19 +2010,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2811,19 +2022,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2831,19 +2034,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2851,16 +2046,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -2869,33 +2057,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -2904,14 +2084,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", @@ -2928,11 +2104,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_04", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -2948,17 +2121,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1588287228997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -2968,11 +2136,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2980,19 +2144,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3000,19 +2156,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3020,19 +2168,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3040,19 +2180,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3060,19 +2192,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3080,16 +2204,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3098,33 +2215,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3133,14 +2242,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", @@ -3157,11 +2262,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_05", - "qualifiedName": null, "description": "a payment", - "uri": null, "tags": [] } }, @@ -3177,17 +2279,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 1589460269997, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3197,11 +2294,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3209,19 +2302,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3229,19 +2314,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3249,19 +2326,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3269,19 +2338,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3289,19 +2350,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3309,16 +2362,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3327,33 +2373,25 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3362,14 +2400,10 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", @@ -3386,11 +2420,8 @@ "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", "catalog_version": "0.19.1" }, - "externalUrl": null, "name": "payment_p2020_06", - "qualifiedName": null, "description": "", - "uri": null, "tags": [] } }, @@ -3406,17 +2437,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null + "actor": "urn:li:corpuser:dbt_executor" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -3426,11 +2452,7 @@ "fields": [ { "fieldPath": "amount", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3438,19 +2460,11 @@ }, "nativeDataType": "numeric(5,2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "customer_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3458,19 +2472,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_date", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3478,19 +2484,11 @@ }, "nativeDataType": "timestamp with time zone", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "payment_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3498,19 +2496,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "rental_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3518,19 +2508,11 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "staff_id", - "jsonPath": null, "nullable": false, - "description": null, - "created": null, - "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3538,16 +2520,9 @@ }, "nativeDataType": "integer", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } }, { @@ -3556,128 +2531,62 @@ { "auditStamp": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } - ], - "fineGrainedLineages": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } }, { - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "value": "[{\"op\": \"add\", \"path\": \"/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"value\": {\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)\", \"type\": \"TRANSFORMED\"}}]", + "contentType": "application/json-patch+json" }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "dbt-test-with-target-platform-instance", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "dbt-test-with-target-platform-instance" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 3ab5d063a60203..34c40f63bcb555 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -157,6 +157,14 @@ def set_paths( "platform_instance": "dbt-instance-1", }, ), + DbtTestConfig( + "dbt-test-with-non-incremental-lineage", + "dbt_test_with_non_incremental_lineage_mces.json", + "dbt_test_with_non_incremental_lineage_mces_golden.json", + source_config_modifiers={ + "incremental_lineage": "False", + }, + ), DbtTestConfig( "dbt-test-with-target-platform-instance", "dbt_test_with_target_platform_instance_mces.json",