From 954397a0785ba4d99da528a431a3015456fef021 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 9 Sep 2022 14:07:55 -0700 Subject: [PATCH] fix(ingest): remove dbt `disable_dbt_node_creation` and `load_schema` options (#5877) --- docs/how/updating-datahub.md | 1 + .../docs/sources/dbt/dbt_recipe.yml | 3 +- .../examples/recipes/dbt_to_datahub.dhub.yaml | 1 - .../src/datahub/ingestion/source/dbt.py | 134 +- ...t_enabled_without_schemas_mces_golden.json | 3683 ----------------- ...thout_schemas_with_filter_mces_golden.json | 3454 ---------------- ...h_external_metadata_files_mces_golden.json | 3057 -------------- .../dbt/dbt_with_schemas_mces_golden.json | 3127 -------------- .../dbt/dbt_without_schemas_mces_golden.json | 935 ----- ...thout_schemas_with_filter_mces_golden.json | 892 ---- .../tests/integration/dbt/test_dbt.py | 92 - 11 files changed, 38 insertions(+), 15341 deletions(-) delete mode 100644 metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json delete mode 100644 metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json delete mode 100644 metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json delete mode 100644 metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json delete mode 100644 metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json delete mode 100644 metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index c81e72be8c126f..1d33fee77c02f1 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -8,6 +8,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - Browse Paths have been upgraded to a new format to align more closely with the intention of the feature. Learn more about the changes, including steps on upgrading, here: https://datahubproject.io/docs/advanced/browse-paths-upgrade +- The dbt ingestion source's `disable_dbt_node_creation` and `load_schema` options have been removed. They were no longer necessary due to the recently added sibling entities functionality. ### Potential Downtime diff --git a/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml b/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml index 4ee236b74b934b..251aba44db3870 100644 --- a/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml +++ b/metadata-ingestion/docs/sources/dbt/dbt_recipe.yml @@ -5,11 +5,10 @@ source: # To use this as-is, set the environment variable DBT_PROJECT_ROOT to the root folder of your dbt project manifest_path: "${DBT_PROJECT_ROOT}/target/manifest_file.json" catalog_path: "${DBT_PROJECT_ROOT}/target/catalog_file.json" - sources_path: "${DBT_PROJECT_ROOT}/target/sources_file.json" # optional for freshness + sources_path: "${DBT_PROJECT_ROOT}/target/sources_file.json" # optional for freshness test_results_path: "${DBT_PROJECT_ROOT}/target/run_results.json" # optional for recording dbt test results after running dbt test # Options target_platform: "my_target_platform_id" # e.g. bigquery/postgres/etc. - load_schemas: False # note: enable this only if you are not ingesting metadata from your warehouse # sink configs diff --git a/metadata-ingestion/examples/recipes/dbt_to_datahub.dhub.yaml b/metadata-ingestion/examples/recipes/dbt_to_datahub.dhub.yaml index 5a37b410111163..15b285e21973a5 100644 --- a/metadata-ingestion/examples/recipes/dbt_to_datahub.dhub.yaml +++ b/metadata-ingestion/examples/recipes/dbt_to_datahub.dhub.yaml @@ -7,7 +7,6 @@ source: catalog_path: "./tests/integration/dbt/dbt_catalog.json" sources_path: "./tests/integration/dbt/dbt_sources.json" target_platform: "dbt" - load_schemas: True # or False # see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation sink: type: "datahub-rest" diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index 021580fd308d49..8ea0be64d3356f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -246,10 +246,6 @@ class DBTConfig(StatefulIngestionConfigBase): default=None, description="The platform instance for the platform that dbt is operating on. Use this if you have multiple instances of the same platform (e.g. redshift) and need to distinguish between them.", ) - load_schemas: bool = Field( - default=True, - description="This flag is only consulted when disable_dbt_node_creation is set to True. Load schemas for target_platform entities from dbt catalog file, not necessary when you are already ingesting this metadata from the data platform directly. If set to False, table schema details (e.g. columns) will not be ingested.", - ) use_identifiers: bool = Field( default=False, description="Use model identifier instead of model name if defined (if not, default to model name).", @@ -269,10 +265,6 @@ class DBTConfig(StatefulIngestionConfigBase): default=AllowDenyPattern.allow_all(), description="regex patterns for dbt model names to filter in ingestion.", ) - disable_dbt_node_creation = Field( - default=False, - description="Whether to suppress dbt dataset metadata creation. When set to True, this flag applies the dbt metadata to the target_platform entities (e.g. populating schema and column descriptions from dbt into the postgres / bigquery table metadata in DataHub) and generates lineage between the platform entities.", - ) meta_mapping: Dict = Field( default={}, description="mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.", @@ -474,7 +466,6 @@ def extract_dbt_entities( all_catalog_entities: Dict[str, Dict[str, Any]], sources_results: List[Dict[str, Any]], manifest_adapter: str, - load_schemas: bool, use_identifiers: bool, tag_prefix: str, node_type_pattern: AllowDenyPattern, @@ -650,7 +641,6 @@ def get_upstreams( target_platform: str, target_platform_instance: Optional[str], environment: str, - disable_dbt_node_creation: bool, platform_instance: Optional[str], legacy_skip_source_lineage: Optional[bool], ) -> List[str]: @@ -672,28 +662,18 @@ def get_upstreams( upstream_manifest_node = all_nodes[upstream] - # This function is called to create lineages among platform nodes or dbt nodes. When we are creating lineages - # for platform nodes, implies that dbt node creation is turned off (because otherwise platform nodes only - # have one lineage edge to their corresponding dbt node). So, when disable_dbt_node_creation is true we only - # create lineages for platform nodes otherwise, for dbt node, we connect it to another dbt node or a platform - # node. + # This logic creates lineages among dbt nodes. platform_value = DBT_PLATFORM platform_instance_value = platform_instance - if disable_dbt_node_creation: - # we generate all urns in the target platform universe + materialized = upstream_manifest_node.get("config", {}).get("materialized") + resource_type = upstream_manifest_node["resource_type"] + if materialized in {"view", "table", "incremental"} or ( + resource_type == "source" and legacy_skip_source_lineage + ): + # upstream urns point to the target platform platform_value = target_platform platform_instance_value = target_platform_instance - else: - materialized = upstream_manifest_node.get("config", {}).get("materialized") - resource_type = upstream_manifest_node["resource_type"] - - if materialized in {"view", "table", "incremental"} or ( - resource_type == "source" and legacy_skip_source_lineage - ): - # upstream urns point to the target platform - platform_value = target_platform - platform_instance_value = target_platform_instance upstream_urns.append( get_urn_from_dbtNode( @@ -989,7 +969,6 @@ def load_test_results( config.target_platform, config.target_platform_instance, config.env, - config.disable_dbt_node_creation, config.platform_instance, config.backcompat_skip_source_on_lineage_edge, ) @@ -1074,8 +1053,7 @@ class DBTSource(StatefulIngestionSourceBase): Note: 1. It also generates lineage between the `dbt` nodes (e.g. ephemeral nodes that depend on other dbt sources) as well as lineage between the `dbt` nodes and the underlying (target) platform nodes (e.g. BigQuery Table -> dbt Source, dbt View -> BigQuery View). - 2. The previous version of this source (`acryl_datahub<=0.8.16.2`) did not generate `dbt` entities and lineage between `dbt` entities and platform entities. For backwards compatibility with the previous version of this source, there is a config flag `disable_dbt_node_creation` that falls back to the old behavior. - 3. We also support automated actions (like add a tag, term or owner) based on properties defined in dbt meta. + 2. We also support automated actions (like add a tag, term or owner) based on properties defined in dbt meta. The artifacts used by this source are: - [dbt manifest file](https://docs.getdbt.com/reference/artifacts/manifest-json) @@ -1214,7 +1192,6 @@ def loadManifestAndCatalog( manifest_path: str, catalog_path: str, sources_path: Optional[str], - load_schemas: bool, use_identifiers: bool, tag_prefix: str, node_type_pattern: AllowDenyPattern, @@ -1263,7 +1240,6 @@ def loadManifestAndCatalog( all_catalog_entities, sources_results, manifest_adapter, - load_schemas, use_identifiers, tag_prefix, node_type_pattern, @@ -1328,7 +1304,6 @@ def string_map(input_map: Dict[str, Any]) -> Dict[str, str]: target_platform=self.config.target_platform, target_platform_instance=self.config.target_platform_instance, environment=self.config.env, - disable_dbt_node_creation=self.config.disable_dbt_node_creation, platform_instance=None, legacy_skip_source_lineage=self.config.backcompat_skip_source_on_lineage_edge, ) @@ -1456,7 +1431,6 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.config.manifest_path, self.config.catalog_path, self.config.sources_path, - self.config.load_schemas, self.config.use_identifiers, self.config.tag_prefix, self.config.node_type_pattern, @@ -1483,14 +1457,13 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: ] test_nodes = [test_node for test_node in nodes if test_node.node_type == "test"] - if not self.config.disable_dbt_node_creation: - yield from self.create_platform_mces( - non_test_nodes, - additional_custom_props_filtered, - manifest_nodes_raw, - DBT_PLATFORM, - self.config.platform_instance, - ) + yield from self.create_platform_mces( + non_test_nodes, + additional_custom_props_filtered, + manifest_nodes_raw, + DBT_PLATFORM, + self.config.platform_instance, + ) yield from self.create_platform_mces( non_test_nodes, @@ -1552,11 +1525,7 @@ def create_platform_mces( This function creates mce based out of dbt nodes. Since dbt ingestion creates "dbt" nodes and nodes for underlying platform the function gets called twice based on the mce_platform parameter. Further, this function takes specific actions based on the mce_platform passed in. - If disable_dbt_node_creation = True, - Create empty entities of the underlying platform with only lineage/key aspect. - Create dbt entities with all metadata information. - If disable_dbt_node_creation = False - Create platform entities with all metadata information. + It creates platform entities with all metadata information. """ action_processor = OperationProcessor( self.config.meta_mapping, @@ -1619,38 +1588,25 @@ def create_platform_mces( self.report.report_workunit(sub_type_wu) else: - if not self.config.disable_dbt_node_creation: - # if dbt node creation is enabled we are creating empty node for platform and only add - # lineage/keyaspect. - aspects = [] - if node.materialization == "ephemeral" or node.node_type == "test": - continue - - # This code block is run when we are generating entities of platform type. - # We will not link the platform not to the dbt node for type "source" because - # in this case the platform table existed first. - if node.node_type != "source": - upstream_dbt_urn = get_urn_from_dbtNode( - node.database, - node.schema, - node.name, - DBT_PLATFORM, - self.config.env, - self.config.platform_instance, - ) - upstreams_lineage_class = get_upstream_lineage( - [upstream_dbt_urn] - ) - aspects.append(upstreams_lineage_class) - else: - # add upstream lineage - platform_upstream_aspect = ( - self._create_lineage_aspect_for_platform_node( - node, manifest_nodes_raw - ) + # We are creating empty node for platform and only add lineage/keyaspect. + aspects = [] + if node.materialization == "ephemeral" or node.node_type == "test": + continue + + # This code block is run when we are generating entities of platform type. + # We will not link the platform not to the dbt node for type "source" because + # in this case the platform table existed first. + if node.node_type != "source": + upstream_dbt_urn = get_urn_from_dbtNode( + node.database, + node.schema, + node.name, + DBT_PLATFORM, + self.config.env, + self.config.platform_instance, ) - if platform_upstream_aspect: - aspects.append(platform_upstream_aspect) + upstreams_lineage_class = get_upstream_lineage([upstream_dbt_urn]) + aspects.append(upstreams_lineage_class) if len(aspects) == 0: continue @@ -1733,16 +1689,7 @@ def get_patched_mce(self, mce): def _create_dataset_properties_aspect( self, node: DBTNode, additional_custom_props_filtered: Dict[str, str] ) -> DatasetPropertiesClass: - description = None - if self.config.disable_dbt_node_creation: - if node.comment and node.description and node.comment != node.description: - description = f"{self.config.target_platform} comment: {node.comment}\n\ndbt model description: {node.description}" - elif node.comment: - description = node.comment - elif node.description: - description = node.description - else: - description = node.description + description = node.description custom_props = { **get_custom_properties(node), @@ -1823,14 +1770,7 @@ def _generate_base_aspects( # add schema metadata aspect schema_metadata = get_schema_metadata(self.report, node, mce_platform) - # When generating these aspects for a dbt node, we will always include schema information. When generating - # these aspects for a platform node (which only happens when disable_dbt_node_creation is set to true) we - # honor the flag. - if mce_platform == DBT_PLATFORM: - aspects.append(schema_metadata) - else: - if self.config.load_schemas: - aspects.append(schema_metadata) + aspects.append(schema_metadata) return aspects def _aggregate_owners( @@ -1915,7 +1855,6 @@ def _create_lineage_aspect_for_dbt_node( self.config.target_platform, self.config.target_platform_instance, self.config.env, - self.config.disable_dbt_node_creation, self.config.platform_instance, self.config.backcompat_skip_source_on_lineage_edge, ) @@ -1951,7 +1890,6 @@ def _create_lineage_aspect_for_platform_node( self.config.target_platform, self.config.target_platform_instance, self.config.env, - self.config.disable_dbt_node_creation, self.config.platform_instance, self.config.backcompat_skip_source_on_lineage_edge, ) diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json deleted file mode 100644 index b0bb78e58475b0..00000000000000 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json +++ /dev/null @@ -1,3683 +0,0 @@ -[ -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "@alice2", - "business_owner": "jdoe.last@gmail.com", - "data_governance.team_owner": "Finance", - "has_pii": "True", - "int_property": "1", - "double_property": "2.5", - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transform/customer_details.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer_details", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [ - "dbt:test_tag" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:test_tag", - "context": null - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.customer_details", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "business_owner": "jdoe.last", - "data_governance.team_owner": "Sales", - "has_pii": "False", - "int_property": "2", - "double_property": "3.5", - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.monthly_billing_with_cust", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": true, - "viewLogic": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/base/payments_base.sql", - "catalog_type": "VIEW", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-payments", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_base", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "{{ config(\n materialized=\"view\",\n alias='an-aliased-view-for-payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payments_by_customer_by_month", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_by_customer_by_month", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": true, - "viewLogic": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n billing_month,\n customer_id", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in dev", - "some_other_property": "test 1", - "owner": "@alice1", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "actor", - "qualifiedName": null, - "description": "description for actor table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.actor", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:column_tag", - "context": null - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "description for last_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "description for last_update from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "address", - "qualifiedName": null, - "description": "a user's address", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.address", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "category", - "qualifiedName": null, - "description": "a user's category", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.category", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "category_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "city", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.city", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "city", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in prod", - "owner": "@bob", - "some_other_property": "test 2", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "country", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.country", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer", - "qualifiedName": null, - "description": "description for customer table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.customer", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "activebool", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "create_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "date", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "store_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_01", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_01", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1580505371996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "an_array_property": "['alpha', 'beta', 'charlie']", - "model_maturity": "in prod", - "owner": "@charles", - "some_other_property": "test 3", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_02", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_02", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1582319845996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_03", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_03", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1584998318996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_04", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_04", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1588287228996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_05", - "qualifiedName": null, - "description": "a payment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_05", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1589460269996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_06", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_06", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json deleted file mode 100644 index fa1a1353e79dc9..00000000000000 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json +++ /dev/null @@ -1,3454 +0,0 @@ -[ -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"ephemeral\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "@alice2", - "business_owner": "jdoe.last@gmail.com", - "data_governance.team_owner": "Finance", - "has_pii": "True", - "int_property": "1", - "double_property": "2.5", - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transform/customer_details.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer_details", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [ - "dbt:test_tag" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:test_tag", - "context": null - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.customer_details", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "business_owner": "jdoe.last", - "data_governance.team_owner": "Sales", - "has_pii": "False", - "int_property": "2", - "double_property": "3.5", - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.monthly_billing_with_cust", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": true, - "viewLogic": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/base/payments_base.sql", - "catalog_type": "VIEW", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-payments", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_base", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "{{ config(\n materialized=\"view\",\n alias='an-aliased-view-for-payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\", \"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payments_by_customer_by_month", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_by_customer_by_month", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": true, - "viewLogic": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n billing_month,\n customer_id", - "viewLanguage": "SQL" - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in dev", - "some_other_property": "test 1", - "owner": "@alice1", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "actor", - "qualifiedName": null, - "description": "description for actor table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.actor", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:column_tag", - "context": null - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "description for last_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "description for last_update from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "address", - "qualifiedName": null, - "description": "a user's address", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.address", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "category", - "qualifiedName": null, - "description": "a user's category", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.category", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "category_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "city", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.city", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "city", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in prod", - "owner": "@bob", - "some_other_property": "test 2", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "country", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.country", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer", - "qualifiedName": null, - "description": "description for customer table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.customer", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "activebool", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "create_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "date", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "store_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_01", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_01", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1580505371996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "an_array_property": "['alpha', 'beta', 'charlie']", - "model_maturity": "in prod", - "owner": "@charles", - "some_other_property": "test 3", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_02", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_02", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1582319845996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_03", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_03", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1584998318996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_04", - "qualifiedName": null, - "description": "", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_04", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1588287228996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"source\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_05", - "qualifiedName": null, - "description": "a payment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_05", - "platform": "urn:li:dataPlatform:dbt", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1589460269996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter-dbt-enabled", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json deleted file mode 100644 index 4b78fa9021c73d..00000000000000 --- a/metadata-ingestion/tests/integration/dbt/dbt_with_external_metadata_files_mces_golden.json +++ /dev/null @@ -1,3057 +0,0 @@ -[ -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "@alice2", - "business_owner": "jdoe.last@gmail.com", - "data_governance.team_owner": "Finance", - "has_pii": "True", - "int_property": "1", - "double_property": "2.5", - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transform/customer_details.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer_details", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [ - "dbt:test_tag" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:test_tag", - "context": null - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.customer_details", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "business_owner": "jdoe.last", - "data_governance.team_owner": "Sales", - "has_pii": "False", - "int_property": "2", - "double_property": "3.5", - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.monthly_billing_with_cust", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/base/payments_base.sql", - "catalog_type": "VIEW", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-payments", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_base", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payments_by_customer_by_month", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_by_customer_by_month", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in dev", - "some_other_property": "test 1", - "owner": "@alice1", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "actor", - "qualifiedName": null, - "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.actor", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:column_tag", - "context": null - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "description for last_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "description for last_update from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "address", - "qualifiedName": null, - "description": "a user's address", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.address", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "category", - "qualifiedName": null, - "description": "a user's category", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.category", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "category_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "city", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.city", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "city", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in prod", - "owner": "@bob", - "some_other_property": "test 2", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "country", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.country", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer", - "qualifiedName": null, - "description": "description for customer table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.customer", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "activebool", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "create_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "date", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "store_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_01", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_01", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1580505371996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "an_array_property": "['alpha', 'beta', 'charlie']", - "model_maturity": "in prod", - "owner": "@charles", - "some_other_property": "test 3", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_02", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_02", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1582319845996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_03", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_03", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1584998318996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_04", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_04", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1588287228996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_05", - "qualifiedName": null, - "description": "a payment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_05", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1589460269996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_06", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_06", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-external-metadata-files", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json deleted file mode 100644 index 36534c8b0b7587..00000000000000 --- a/metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json +++ /dev/null @@ -1,3127 +0,0 @@ -[ -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "@alice2", - "business_owner": "jdoe.last@gmail.com", - "data_governance.team_owner": "Finance", - "has_pii": "True", - "int_property": "1", - "double_property": "2.5", - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transform/customer_details.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer_details", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [ - "dbt:test_tag" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": { - "type": "SOURCE_CONTROL", - "url": null - } - }, - { - "owner": "urn:li:corpuser:jdoe.last@gmail.com", - "type": "DATAOWNER", - "source": { - "type": "SOURCE_CONTROL", - "url": null - } - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:has_pii_test", - "context": null - }, - { - "tag": "urn:li:tag:dbt:int_meta_property", - "context": null - }, - { - "tag": "urn:li:tag:dbt:my_query_tag", - "context": null - }, - { - "tag": "urn:li:tag:dbt:test_tag", - "context": null - } - ] - } - }, - { - "com.linkedin.pegasus2avro.common.GlossaryTerms": { - "terms": [ - { - "urn": "urn:li:glossaryTerm:Finance_test", - "context": null - }, - { - "urn": "urn:li:glossaryTerm:double_meta_property", - "context": null - } - ], - "auditStamp": { - "time": 1615443388097, - "actor": "urn:li:corpuser:datahub", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.customer_details", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "business_owner": "jdoe.last", - "data_governance.team_owner": "Sales", - "has_pii": "False", - "int_property": "2", - "double_property": "3.5", - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:jdoe.last", - "type": "DATAOWNER", - "source": { - "type": "SOURCE_CONTROL", - "url": null - } - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.monthly_billing_with_cust", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/base/payments_base.sql", - "catalog_type": "VIEW", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-payments", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_base", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payments_by_customer_by_month", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "model.sample_dbt.payments_by_customer_by_month", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "billing_month", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in dev", - "some_other_property": "test 1", - "owner": "@alice1", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "actor", - "qualifiedName": null, - "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": { - "type": "SOURCE_CONTROL", - "url": null - } - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.actor", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759273000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "actor_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:column_tag", - "context": null - } - ] - }, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": "description for last_name from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": "description for last_update from dbt", - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "address", - "qualifiedName": null, - "description": "a user's address", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.address", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759930000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "address", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address2", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "district", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "phone", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "postal_code", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "category", - "qualifiedName": null, - "description": "a user's category", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.category", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759987000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "category_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "city", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.city", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759925000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "city", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "city_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in prod", - "owner": "@bob", - "some_other_property": "test 2", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "country", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": { - "type": "SOURCE_CONTROL", - "url": null - } - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.country", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581759840000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "country", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "country_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer", - "qualifiedName": null, - "description": "description for customer table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.customer", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1581760640000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "active", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "activebool", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.BooleanType": {} - } - }, - "nativeDataType": "boolean", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "address_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "create_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} - } - }, - "nativeDataType": "date", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "email", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "first_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_name", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "text", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "last_update", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "store_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_01", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_01", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1580505371996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "an_array_property": "['alpha', 'beta', 'charlie']", - "model_maturity": "in prod", - "owner": "@charles", - "some_other_property": "test 3", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_02", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": { - "type": "SOURCE_CONTROL", - "url": null - } - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_02", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1582319845996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_03", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_03", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1584998318996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_04", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_04", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1588287228996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_05", - "qualifiedName": null, - "description": "a payment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_05", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 1589460269996, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_06", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "source.sample_dbt.pagila.payment_p2020_06", - "platform": "urn:li:dataPlatform:postgres", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": -62135596800000, - "actor": "urn:li:corpuser:dbt_executor", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "amount", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "numeric(5,2)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "customer_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_date", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.TimeType": {} - } - }, - "nativeDataType": "timestamp with time zone", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "payment_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "rental_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - }, - { - "fieldPath": "staff_id", - "jsonPath": null, - "nullable": false, - "description": null, - "created": null, - "lastModified": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-with-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json deleted file mode 100644 index 04e0cacfb313e5..00000000000000 --- a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json +++ /dev/null @@ -1,935 +0,0 @@ -[ -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "@alice2", - "business_owner": "jdoe.last@gmail.com", - "data_governance.team_owner": "Finance", - "has_pii": "True", - "int_property": "1", - "double_property": "2.5", - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transform/customer_details.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer_details", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [ - "dbt:test_tag" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:test_tag", - "context": null - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "business_owner": "jdoe.last", - "data_governance.team_owner": "Sales", - "has_pii": "False", - "int_property": "2", - "double_property": "3.5", - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/base/payments_base.sql", - "catalog_type": "VIEW", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-payments", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payments_by_customer_by_month", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in dev", - "some_other_property": "test 1", - "owner": "@alice1", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "actor", - "qualifiedName": null, - "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "address", - "qualifiedName": null, - "description": "a user's address", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "category", - "qualifiedName": null, - "description": "a user's category", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "city", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in prod", - "owner": "@bob", - "some_other_property": "test 2", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "country", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer", - "qualifiedName": null, - "description": "description for customer table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_01", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "an_array_property": "['alpha', 'beta', 'charlie']", - "model_maturity": "in prod", - "owner": "@charles", - "some_other_property": "test 3", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_02", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_03", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_04", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_05", - "qualifiedName": null, - "description": "a payment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_06", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json deleted file mode 100644 index b2af77796ec5f9..00000000000000 --- a/metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json +++ /dev/null @@ -1,892 +0,0 @@ -[ -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "owner": "@alice2", - "business_owner": "jdoe.last@gmail.com", - "data_governance.team_owner": "Finance", - "has_pii": "True", - "int_property": "1", - "double_property": "2.5", - "node_type": "model", - "materialization": "ephemeral", - "dbt_file_path": "models/transform/customer_details.sql", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer_details", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [ - "dbt:test_tag" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice2", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - }, - { - "com.linkedin.pegasus2avro.common.GlobalTags": { - "tags": [ - { - "tag": "urn:li:tag:dbt:test_tag", - "context": null - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "business_owner": "jdoe.last", - "data_governance.team_owner": "Sales", - "has_pii": "False", - "int_property": "2", - "double_property": "3.5", - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-monthly-billing", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "view", - "dbt_file_path": "models/base/payments_base.sql", - "catalog_type": "VIEW", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "an-aliased-view-for-payments", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "type": "TRANSFORMED" - }, - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "model", - "materialization": "table", - "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payments_by_customer_by_month", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", - "type": "TRANSFORMED" - } - ], - "fineGrainedLineages": null - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in dev", - "some_other_property": "test 1", - "owner": "@alice1", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "actor", - "qualifiedName": null, - "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:alice1", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "address", - "qualifiedName": null, - "description": "a user's address", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "category", - "qualifiedName": null, - "description": "a user's category", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "city", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "model_maturity": "in prod", - "owner": "@bob", - "some_other_property": "test 2", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "country", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:bob", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "customer", - "qualifiedName": null, - "description": "description for customer table from dbt", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_01", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "an_array_property": "['alpha', 'beta', 'charlie']", - "model_maturity": "in prod", - "owner": "@charles", - "some_other_property": "test 3", - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_02", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.common.Ownership": { - "owners": [ - { - "owner": "urn:li:corpuser:charles", - "type": "DATAOWNER", - "source": null - } - ], - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - } - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_03", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_04", - "qualifiedName": null, - "description": null, - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "node_type": "source", - "dbt_file_path": "models/base.yml", - "catalog_type": "BASE TABLE", - "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "manifest_version": "0.19.1", - "manifest_adapter": "postgres", - "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "catalog_version": "0.19.1" - }, - "externalUrl": null, - "name": "payment_p2020_05", - "qualifiedName": null, - "description": "a payment", - "uri": null, - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - } - ] - } - }, - "proposedDelta": null, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "dbt-test-without-schemas-with-filter", - "registryName": null, - "registryVersion": null, - "properties": null - } -} -] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 20abea5b91509a..c12dbb61292898 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -145,62 +145,6 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): ) config_variants = [ - DbtTestConfig( - "dbt-test-with-schemas", - test_resources_dir, - test_resources_dir, - tmp_path, - "dbt_with_schemas_mces.json", - "dbt_with_schemas_mces_golden.json", - source_config_modifiers={ - "load_schemas": True, - "disable_dbt_node_creation": True, - "enable_meta_mapping": True, - "owner_extraction_pattern": r"^@(?P(.*))", - }, - ), - DbtTestConfig( - "dbt-test-with-external-metadata-files", - "http://some-external-repo", - test_resources_dir, - tmp_path, - "dbt_with_external_metadata_files_mces.json", - "dbt_with_external_metadata_files_mces_golden.json", - source_config_modifiers={ - "load_schemas": True, - "disable_dbt_node_creation": True, - "owner_extraction_pattern": r"^@(?P(.*))", - }, - ), - DbtTestConfig( - "dbt-test-without-schemas", - test_resources_dir, - test_resources_dir, - tmp_path, - "dbt_without_schemas_mces.json", - "dbt_without_schemas_mces_golden.json", - source_config_modifiers={ - "load_schemas": False, - "disable_dbt_node_creation": True, - "owner_extraction_pattern": r"^@(?P(.*))", - }, - ), - DbtTestConfig( - "dbt-test-without-schemas-with-filter", - test_resources_dir, - test_resources_dir, - tmp_path, - "dbt_without_schemas_with_filter_mces.json", - "dbt_without_schemas_with_filter_mces_golden.json", - source_config_modifiers={ - "load_schemas": False, - "node_name_pattern": { - "deny": ["source.sample_dbt.pagila.payment_p2020_06"] - }, - "disable_dbt_node_creation": True, - "owner_extraction_pattern": r"^@(?P(.*))", - }, - ), DbtTestConfig( "dbt-test-with-schemas-dbt-enabled", test_resources_dir, @@ -209,38 +153,10 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): "dbt_enabled_with_schemas_mces.json", "dbt_enabled_with_schemas_mces_golden.json", source_config_modifiers={ - "load_schemas": True, "enable_meta_mapping": True, "owner_extraction_pattern": r"^@(?P(.*))", }, ), - DbtTestConfig( - "dbt-test-without-schemas-dbt-enabled", - test_resources_dir, - test_resources_dir, - tmp_path, - "dbt_enabled_without_schemas_mces.json", - "dbt_enabled_without_schemas_mces_golden.json", - source_config_modifiers={ - "load_schemas": False, - "owner_extraction_pattern": r"^@(?P(.*))", - }, - ), - DbtTestConfig( - "dbt-test-without-schemas-with-filter-dbt-enabled", - test_resources_dir, - test_resources_dir, - tmp_path, - "dbt_enabled_without_schemas_with_filter_mces.json", - "dbt_enabled_without_schemas_with_filter_mces_golden.json", - source_config_modifiers={ - "load_schemas": False, - "node_name_pattern": { - "deny": ["source.sample_dbt.pagila.payment_p2020_06"] - }, - "owner_extraction_pattern": r"^@(?P(.*))", - }, - ), DbtTestConfig( "dbt-test-with-complex-owner-patterns", test_resources_dir, @@ -250,7 +166,6 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): "dbt_test_with_complex_owner_patterns_mces_golden.json", manifest_file="dbt_manifest_complex_owner_patterns.json", source_config_modifiers={ - "load_schemas": False, "node_name_pattern": { "deny": ["source.sample_dbt.pagila.payment_p2020_06"] }, @@ -266,8 +181,6 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): "dbt_test_with_data_platform_instance_mces.json", "dbt_test_with_data_platform_instance_mces_golden.json", source_config_modifiers={ - "load_schemas": True, - "disable_dbt_node_creation": False, "platform_instance": "dbt-instance-1", }, ), @@ -279,7 +192,6 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time, **kwargs): "dbt_test_with_target_platform_instance_mces.json", "dbt_test_with_target_platform_instance_mces_golden.json", source_config_modifiers={ - "load_schemas": True, "target_platform_instance": "ps-instance-1", }, ), @@ -352,7 +264,6 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): "catalog_path": catalog_path, "sources_path": sources_path, "target_platform": "postgres", - "load_schemas": True, # This will bypass check in get_workunits function of dbt.py "write_semantics": "OVERRIDE", "owner_extraction_pattern": r"^@(?P(.*))", @@ -365,7 +276,6 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): "catalog_path": catalog_path_deleted_actor, "sources_path": sources_path_deleted_actor, "target_platform": "postgres", - "load_schemas": True, "write_semantics": "OVERRIDE", "owner_extraction_pattern": r"^@(?P(.*))", # enable stateful ingestion @@ -467,7 +377,6 @@ def test_dbt_state_backward_compatibility( "catalog_path": catalog_path, "sources_path": sources_path, "target_platform": "postgres", - "load_schemas": True, # This will bypass check in get_workunits function of dbt.py "write_semantics": "OVERRIDE", "owner_extraction_pattern": r"^@(?P(.*))", @@ -607,7 +516,6 @@ def test_dbt_stateful_tests(pytestconfig, tmp_path, mock_time, mock_datahub_grap "catalog_path": catalog_path, "test_results_path": test_results_path, "target_platform": "postgres", - "load_schemas": True, # This will bypass check in get_workunits function of dbt.py "write_semantics": "OVERRIDE", "owner_extraction_pattern": r"^@(?P(.*))",