From 0545f3cb380e19a0a33e3c6e4bd2c373a5df6fe8 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 19 Oct 2022 18:28:33 +0200 Subject: [PATCH] fix(ingest): presto-on-hive - Adding catalog name to the presto on hive urn (#6024) Co-authored-by: Shirshanka Das --- .../ingestion/source/sql/presto_on_hive.py | 33 +- .../presto_on_hive_mces_golden_1.json | 2432 ++++++++--------- .../presto_on_hive_mces_golden_2.json | 2166 +++++++-------- .../presto_on_hive_mces_golden_3.json | 1337 +++++++++ .../presto_on_hive_mces_golden_4.json | 1191 ++++++++ .../presto-on-hive/test_presto_on_hive.py | 10 +- 6 files changed, 4864 insertions(+), 2305 deletions(-) create mode 100644 metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json create mode 100644 metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index 2350c7f915887..d9274246a5a34 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -121,6 +121,11 @@ class PrestoOnHiveConfig(BasicSQLAlchemyConfig): description="Dataset Subtype name to be 'Table' or 'View' Valid options: ['True', 'False']", ) + include_catalog_name_in_ids: bool = Field( + default=False, + description="Add the Presto catalog name (e.g. hive) to the generated dataset urns. `urn:li:dataset:(urn:li:dataPlatform:hive,hive.user.logging_events,PROD)` versus `urn:li:dataset:(urn:li:dataPlatform:hive,user.logging_events,PROD)`", + ) + def get_sql_alchemy_url(self, uri_opts: Optional[Dict[str, Any]] = None) -> str: if not ((self.host_port and self.scheme) or self.sqlalchemy_uri): raise ValueError("host_port and schema or connect_uri required.") @@ -407,9 +412,17 @@ def loop_tables( iter_res = self._alchemy_client.execute_query(statement) for key, group in groupby(iter_res, self._get_table_key): + db_name = self.get_db_name(inspector) + schema_name = ( + f"{db_name}.{key.schema}" + if self.config.include_catalog_name_in_ids + else key.schema + ) + dataset_name = self.get_identifier( - schema=key.schema, entity=key.table, inspector=inspector + schema=schema_name, entity=key.table, inspector=inspector ) + self.report.report_entity_scanned(dataset_name, ent_type="table") if not sql_config.table_pattern.allowed(dataset_name): @@ -521,8 +534,14 @@ def get_hive_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset]: iter_res = self._alchemy_client.execute_query(statement) for key, group in groupby(iter_res, self._get_table_key): + db_name = self.get_db_name(inspector) + schema_name = ( + f"{db_name}.{key.schema}" + if self.config.include_catalog_name_in_ids + else key.schema + ) dataset_name = self.get_identifier( - schema=key.schema, entity=key.table, inspector=inspector + schema=schema_name, entity=key.table, inspector=inspector ) columns = list(group) @@ -553,8 +572,16 @@ def get_presto_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset] iter_res = self._alchemy_client.execute_query(statement) for row in iter_res: + db_name = self.get_db_name(inspector) + schema_name = ( + f"{db_name}.{row['schema']}" + if self.config.include_catalog_name_in_ids + else row["schema"] + ) dataset_name = self.get_identifier( - schema=row["schema"], entity=row["name"], inspector=inspector + schema=schema_name, + entity=row["name"], + inspector=inspector, ) columns, view_definition = self._get_presto_view_column_metadata( diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json index 70fa2eefac458..4c9c0d6326a5a 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json @@ -1,1337 +1,1337 @@ [ -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.map_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=string].keyvalue", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.map_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", - "nullable": true, - "type": { + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.MapType": { - "keyType": "string", - "valueType": "string" + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "map", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" - }, - "name": "map_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.union_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=union].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } - }, - "nativeDataType": "union", - "recursive": false, - "isPartOfKey": false + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.union_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "struct0", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" - }, - "name": "union_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.nested_struct_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.nested_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "struct", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" }, - "nativeDataType": "varchar(50)", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" }, - "nativeDataType": "tinyint", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" - }, - "name": "nested_struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "description": "id of property", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", - "nullable": true, - "description": "service types and providers", - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array>>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" - }, - "name": "array_struct_test", - "description": "This table has array of structs", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.struct_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" - }, - "name": "struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1._test_table_underscore", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1._test_table_underscore", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" - }, - "name": "_test_table_underscore", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.pokes", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.pokes", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=string].baz", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", - "partitioned_columns": "baz" - }, - "name": "pokes", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test_presto_view", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=string].dummy", - "nullable": true, - "type": { + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].dummy", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "is_view": "True" - }, - "name": "array_struct_test_presto_view", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "/* Presto View */", - "viewLanguage": "SQL" + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "/* Presto View */", + "viewLanguage": "SQL" + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"/* Presto View */\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"/* Presto View */\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test_view", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array>>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "is_view": "True" - }, - "name": "array_struct_test_view", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", - "viewLanguage": "SQL" + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", + "viewLanguage": "SQL" + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } -} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json index d3970673f14ba..dfd6c462e3be0 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json @@ -1,1191 +1,1191 @@ [ -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Catalog\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Catalog\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:e998a77f6edaa92d1326dec9d37c96ab\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:e998a77f6edaa92d1326dec9d37c96ab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.map_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=string].keyvalue", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.map_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.MapType": { - "keyType": "string", - "valueType": "string" + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "map", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" - }, - "name": "map_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.union_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=union].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } - }, - "nativeDataType": "union", - "recursive": false, - "isPartOfKey": false + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.union_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", - "nullable": true, - "type": { + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "struct0", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" - }, - "name": "union_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.nested_struct_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.nested_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "struct", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" }, - "nativeDataType": "varchar(50)", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" }, - "nativeDataType": "tinyint", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" - }, - "name": "nested_struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "description": "id of property", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", - "nullable": true, - "description": "service types and providers", - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array>>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" - }, - "name": "array_struct_test", - "description": "This table has array of structs", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.struct_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" - }, - "name": "struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1._test_table_underscore", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1._test_table_underscore", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" - }, - "name": "_test_table_underscore", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.pokes", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.pokes", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=string].baz", - "nullable": true, - "type": { + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", - "partitioned_columns": "baz" - }, - "name": "pokes", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test_presto_view", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=null].service", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, - "nativeDataType": "array(row(\"type\" varchar,\"provider\" array(integer)))", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array(row(\\\"type\\\" varchar,\\\"provider\\\" array(integer)))\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "is_view": "True" - }, - "name": "array_struct_test_presto_view", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "SELECT *\nFROM\n db1.array_struct_test\n", - "viewLanguage": "SQL" + { + "fieldPath": "[version=2.0].[type=struct].[type=null].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "array(row(\"type\" varchar,\"provider\" array(integer)))", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array(row(\\\"type\\\" varchar,\\\"provider\\\" array(integer)))\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT *\nFROM\n db1.array_struct_test\n", + "viewLanguage": "SQL" + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"View\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"SELECT *\\nFROM\\n db1.array_struct_test\\n\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"View\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"SELECT *\\nFROM\\n db1.array_struct_test\\n\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } -} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json new file mode 100644 index 0000000000000..e016735fde90f --- /dev/null +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json @@ -0,0 +1,1337 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.map_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.union_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.nested_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1._test_table_underscore", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.pokes", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].dummy", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "/* Presto View */", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"/* Presto View */\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json new file mode 100644 index 0000000000000..f0915dcdf26cf --- /dev/null +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json @@ -0,0 +1,1191 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Catalog\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:e998a77f6edaa92d1326dec9d37c96ab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.map_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.union_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.nested_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1._test_table_underscore", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.pokes", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=null].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "array(row(\"type\" varchar,\"provider\" array(integer)))", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array(row(\\\"type\\\" varchar,\\\"provider\\\" array(integer)))\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT *\nFROM\n db1.array_struct_test\n", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"View\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"SELECT *\\nFROM\\n db1.array_struct_test\\n\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index 315f49a10b218..e4f904ab91e74 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -53,10 +53,12 @@ def loaded_presto_on_hive(presto_on_hive_runner): @freeze_time(FROZEN_TIME) @pytest.mark.integration_batch_1 @pytest.mark.parametrize( - "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,test_suffix", + "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,include_catalog_name_in_ids,test_suffix", [ - ("hive", False, False, "_1"), - ("presto-on-hive", True, True, "_2"), + ("hive", False, False, False, "_1"), + ("presto-on-hive", True, True, False, "_2"), + ("hive", False, False, True, "_3"), + ("presto-on-hive", True, True, True, "_4"), ], ) def test_presto_on_hive_ingest( @@ -68,6 +70,7 @@ def test_presto_on_hive_ingest( mode, use_catalog_subtype, use_dataset_pascalcase_subtype, + include_catalog_name_in_ids, test_suffix, ): @@ -91,6 +94,7 @@ def test_presto_on_hive_ingest( "scheme": "postgresql+psycopg2", "include_views": True, "include_tables": True, + "include_catalog_name_in_ids": include_catalog_name_in_ids, "schema_pattern": {"allow": ["^public"]}, "mode": mode, "use_catalog_subtype": use_catalog_subtype,