From 4b2ef38c6d10fe90c8763070c373976d629e3eca Mon Sep 17 00:00:00 2001 From: treff7es Date: Thu, 22 Sep 2022 17:19:20 +0200 Subject: [PATCH 1/4] Adding db name to the presto on hive urn --- .../ingestion/source/sql/presto_on_hive.py | 11 ++- .../presto_on_hive_mces_golden_1.json | 90 +++++++++---------- .../presto_on_hive_mces_golden_2.json | 80 ++++++++--------- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index 1bad0e0a1be740..8aafbc3fa147b1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -405,8 +405,9 @@ def loop_tables( iter_res = self._alchemy_client.execute_query(statement) for key, group in groupby(iter_res, self._get_table_key): + db_name = self.get_db_name(inspector) dataset_name = self.get_identifier( - schema=key.schema, entity=key.table, inspector=inspector + schema=f"{db_name}.{key.schema}", entity=key.table, inspector=inspector ) self.report.report_entity_scanned(dataset_name, ent_type="table") @@ -518,8 +519,9 @@ def get_hive_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset]: iter_res = self._alchemy_client.execute_query(statement) for key, group in groupby(iter_res, self._get_table_key): + db_name = self.get_db_name(inspector) dataset_name = self.get_identifier( - schema=key.schema, entity=key.table, inspector=inspector + schema=f"{db_name}.{key.schema}", entity=key.table, inspector=inspector ) columns = list(group) @@ -550,8 +552,11 @@ def get_presto_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset] iter_res = self._alchemy_client.execute_query(statement) for row in iter_res: + db_name = self.get_db_name(inspector) dataset_name = self.get_identifier( - schema=row["schema"], entity=row["name"], inspector=inspector + schema=f"{db_name}.{row['schema']}", + entity=row["name"], + inspector=inspector, ) columns, view_definition = self._get_presto_view_column_metadata( diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json index 1b0c2609e5590d..05ea86e7534ebe 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json @@ -99,7 +99,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -114,7 +114,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -123,7 +123,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.map_test", + "schemaName": "hive.db1.map_test", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -176,7 +176,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, @@ -193,7 +193,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -207,7 +207,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -222,7 +222,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -231,7 +231,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.union_test", + "schemaName": "hive.db1.union_test", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -341,7 +341,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, @@ -358,7 +358,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -372,7 +372,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -387,7 +387,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -396,7 +396,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.nested_struct_test", + "schemaName": "hive.db1.nested_struct_test", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -498,7 +498,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, @@ -515,7 +515,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -529,7 +529,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -544,7 +544,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -553,7 +553,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test", + "schemaName": "hive.db1.array_struct_test", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -639,7 +639,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, @@ -657,7 +657,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -671,7 +671,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -686,7 +686,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -695,7 +695,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.struct_test", + "schemaName": "hive.db1.struct_test", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -775,7 +775,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, @@ -792,7 +792,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -806,7 +806,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -821,7 +821,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -830,7 +830,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1._test_table_underscore", + "schemaName": "hive.db1._test_table_underscore", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -880,7 +880,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, @@ -897,7 +897,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -911,7 +911,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -926,7 +926,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -935,7 +935,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.pokes", + "schemaName": "hive.db1.pokes", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -998,7 +998,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" @@ -1016,7 +1016,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1030,7 +1030,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1045,7 +1045,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1054,7 +1054,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test_presto_view", + "schemaName": "hive.db1.array_struct_test_presto_view", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -1113,7 +1113,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1127,7 +1127,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { @@ -1141,7 +1141,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1156,7 +1156,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1165,7 +1165,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test_view", + "schemaName": "hive.db1.array_struct_test_view", "platform": "urn:li:dataPlatform:hive", "version": 0, "created": { @@ -1271,7 +1271,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1285,7 +1285,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json index c2403ae18bf753..49ace484817a0a 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json @@ -99,7 +99,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -114,7 +114,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -123,7 +123,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.map_test", + "schemaName": "hive.db1.map_test", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -176,7 +176,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" }, @@ -193,7 +193,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -207,7 +207,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -222,7 +222,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -231,7 +231,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.union_test", + "schemaName": "hive.db1.union_test", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -341,7 +341,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" }, @@ -358,7 +358,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -372,7 +372,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -387,7 +387,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -396,7 +396,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.nested_struct_test", + "schemaName": "hive.db1.nested_struct_test", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -498,7 +498,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" }, @@ -515,7 +515,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -529,7 +529,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -544,7 +544,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -553,7 +553,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test", + "schemaName": "hive.db1.array_struct_test", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -639,7 +639,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" }, @@ -657,7 +657,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -671,7 +671,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -686,7 +686,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -695,7 +695,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.struct_test", + "schemaName": "hive.db1.struct_test", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -775,7 +775,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" }, @@ -792,7 +792,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -806,7 +806,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -821,7 +821,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -830,7 +830,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1._test_table_underscore", + "schemaName": "hive.db1._test_table_underscore", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -880,7 +880,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" }, @@ -897,7 +897,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -911,7 +911,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -926,7 +926,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -935,7 +935,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.pokes", + "schemaName": "hive.db1.pokes", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -998,7 +998,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_date": "2022-09-20", + "create_date": "2022-09-22", "table_type": "MANAGED_TABLE", "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", "partitioned_columns": "baz" @@ -1016,7 +1016,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1030,7 +1030,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { @@ -1045,7 +1045,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -1054,7 +1054,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "db1.array_struct_test_presto_view", + "schemaName": "hive.db1.array_struct_test_presto_view", "platform": "urn:li:dataPlatform:presto-on-hive", "version": 0, "created": { @@ -1126,7 +1126,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -1140,7 +1140,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { From 7d5dc3eabc449bd00f9225737a59ade9c89bb5e5 Mon Sep 17 00:00:00 2001 From: treff7es Date: Tue, 18 Oct 2022 22:29:04 +0200 Subject: [PATCH 2/4] Making configurable to add database name to urns --- .../ingestion/source/sql/presto_on_hive.py | 28 +- .../presto_on_hive_mces_golden_1.json | 2432 ++++++++--------- .../presto_on_hive_mces_golden_2.json | 2166 +++++++-------- .../presto_on_hive_mces_golden_3.json | 1337 +++++++++ .../presto_on_hive_mces_golden_4.json | 1191 ++++++++ .../presto-on-hive/test_presto_on_hive.py | 4 + 6 files changed, 4856 insertions(+), 2302 deletions(-) create mode 100644 metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json create mode 100644 metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index f95a9980edde1d..c0389a23768cff 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -121,6 +121,11 @@ class PrestoOnHiveConfig(BasicSQLAlchemyConfig): description="Dataset Subtype name to be 'Table' or 'View' Valid options: ['True', 'False']", ) + add_db_name_to_urns: bool = Field( + default=False, + description="Add database name/alias to the generated dataset urns", + ) + def get_sql_alchemy_url(self, uri_opts: Optional[Dict[str, Any]] = None) -> str: if not ((self.host_port and self.scheme) or self.sqlalchemy_uri): raise ValueError("host_port and schema or connect_uri required.") @@ -408,9 +413,16 @@ def loop_tables( for key, group in groupby(iter_res, self._get_table_key): db_name = self.get_db_name(inspector) + schema_name = ( + f"{db_name}.{key.schema}" + if self.config.add_db_name_to_urns + else key.schema + ) + dataset_name = self.get_identifier( - schema=f"{db_name}.{key.schema}", entity=key.table, inspector=inspector + schema=schema_name, entity=key.table, inspector=inspector ) + self.report.report_entity_scanned(dataset_name, ent_type="table") if not sql_config.table_pattern.allowed(dataset_name): @@ -523,8 +535,13 @@ def get_hive_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset]: iter_res = self._alchemy_client.execute_query(statement) for key, group in groupby(iter_res, self._get_table_key): db_name = self.get_db_name(inspector) + schema_name = ( + f"{db_name}.{key.schema}" + if self.config.add_db_name_to_urns + else key.schema + ) dataset_name = self.get_identifier( - schema=f"{db_name}.{key.schema}", entity=key.table, inspector=inspector + schema=schema_name, entity=key.table, inspector=inspector ) columns = list(group) @@ -556,8 +573,13 @@ def get_presto_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset] iter_res = self._alchemy_client.execute_query(statement) for row in iter_res: db_name = self.get_db_name(inspector) + schema_name = ( + f"{db_name}.{row['schema']}" + if self.config.add_db_name_to_urns + else row["schema"] + ) dataset_name = self.get_identifier( - schema=f"{db_name}.{row['schema']}", + schema=schema_name, entity=row["name"], inspector=inspector, ) diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json index e016735fde90fe..4c9c0d6326a5aa 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_1.json @@ -1,1337 +1,1337 @@ [ -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.map_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=string].keyvalue", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.map_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", - "nullable": true, - "type": { + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.MapType": { - "keyType": "string", - "valueType": "string" + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "map", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" - }, - "name": "map_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.union_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=union].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } - }, - "nativeDataType": "union", - "recursive": false, - "isPartOfKey": false + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.union_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "struct0", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" - }, - "name": "union_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.nested_struct_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.nested_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "struct", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" }, - "nativeDataType": "varchar(50)", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" }, - "nativeDataType": "tinyint", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" - }, - "name": "nested_struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.array_struct_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "description": "id of property", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", - "nullable": true, - "description": "service types and providers", - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array>>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" - }, - "name": "array_struct_test", - "description": "This table has array of structs", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.struct_test", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" - }, - "name": "struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1._test_table_underscore", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1._test_table_underscore", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" - }, - "name": "_test_table_underscore", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.pokes", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.pokes", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=string].baz", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", - "partitioned_columns": "baz" - }, - "name": "pokes", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.array_struct_test_presto_view", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=string].dummy", - "nullable": true, - "type": { + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].dummy", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "is_view": "True" - }, - "name": "array_struct_test_presto_view", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "/* Presto View */", - "viewLanguage": "SQL" + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "/* Presto View */", + "viewLanguage": "SQL" + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"/* Presto View */\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"/* Presto View */\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.array_struct_test_view", - "platform": "urn:li:dataPlatform:hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array>>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "is_view": "True" - }, - "name": "array_struct_test_view", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", - "viewLanguage": "SQL" + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", + "viewLanguage": "SQL" + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"view\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } -} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json index f0915dcdf26cf7..dfd6c462e3be03 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_2.json @@ -1,1191 +1,1191 @@ [ -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Catalog\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Catalog\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "value": "{\"removed\": false}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:e998a77f6edaa92d1326dec9d37c96ab\"}", - "contentType": "application/json" + { + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:e998a77f6edaa92d1326dec9d37c96ab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.map_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=string].keyvalue", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.map_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.MapType": { - "keyType": "string", - "valueType": "string" + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "map", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" - }, - "name": "map_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.union_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=struct].[type=union].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } - }, - "nativeDataType": "union", - "recursive": false, - "isPartOfKey": false + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.union_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", - "nullable": true, - "type": { + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "double", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.UnionType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "struct0", - "recursive": false, - "isPartOfKey": false - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" - }, - "name": "union_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.nested_struct_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.nested_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "struct", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" }, - "nativeDataType": "varchar(50)", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" }, - "nativeDataType": "tinyint", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" - }, - "name": "nested_struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.array_struct_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "description": "id of property", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", - "nullable": true, - "description": "service types and providers", - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "record" - ] + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } - } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "array>>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array>>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" - }, - "name": "array_struct_test", - "description": "This table has array of structs", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.struct_test", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.RecordType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "struct>", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"struct>\"}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.ArrayType": { - "nestedType": [ - "int" - ] + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } - } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "array", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array\"}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" - }, - "name": "struct_test", - "tags": [] + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1._test_table_underscore", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1._test_table_underscore", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" - }, - "name": "_test_table_underscore", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.pokes", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].foo", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "int", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.pokes", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=string].baz", - "nullable": true, - "type": { + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - }, - { - "fieldPath": "[version=2.0].[type=string].bar", - "nullable": true, - "type": { + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, - "nativeDataType": "string", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "create_date": "2022-10-08", - "table_type": "MANAGED_TABLE", - "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", - "partitioned_columns": "baz" - }, - "name": "pokes", - "tags": [] + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "hive.db1.array_struct_test_presto_view", - "platform": "urn:li:dataPlatform:presto-on-hive", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "[version=2.0].[type=int].property_id", - "nullable": true, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "integer", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + { + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - { - "fieldPath": "[version=2.0].[type=struct].[type=null].service", - "nullable": true, - "type": { + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} - } + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, - "nativeDataType": "array(row(\"type\" varchar,\"provider\" array(integer)))", - "recursive": false, - "isPartOfKey": false, - "jsonProps": "{\"native_data_type\": \"array(row(\\\"type\\\" varchar,\\\"provider\\\" array(integer)))\", \"_nullable\": true}" - } - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "is_view": "True" - }, - "name": "array_struct_test_presto_view", - "tags": [] - } - }, - { - "com.linkedin.pegasus2avro.dataset.ViewProperties": { - "materialized": false, - "viewLogic": "SELECT *\nFROM\n db1.array_struct_test\n", - "viewLanguage": "SQL" + { + "fieldPath": "[version=2.0].[type=struct].[type=null].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "array(row(\"type\" varchar,\"provider\" array(integer)))", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array(row(\\\"type\\\" varchar,\\\"provider\\\" array(integer)))\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT *\nFROM\n db1.array_struct_test\n", + "viewLanguage": "SQL" + } } - } - ] + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "value": "{\"typeNames\": [\"View\"]}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "value": "{\"materialized\": false, \"viewLogic\": \"SELECT *\\nFROM\\n db1.array_struct_test\\n\", \"viewLanguage\": \"SQL\"}", - "contentType": "application/json" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"View\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "presto-on-hive-test" + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"SELECT *\\nFROM\\n db1.array_struct_test\\n\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } } -} ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json new file mode 100644 index 00000000000000..e016735fde90fe --- /dev/null +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_3.json @@ -0,0 +1,1337 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f5e571e4a9acce86333e6b427ba1651f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:939ecec0f01fb6bb1ca15fe6f0ead918\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.map_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.union_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.nested_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.struct_test", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1._test_table_underscore", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.pokes", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].dummy", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "/* Presto View */", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"/* Presto View */\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f5e571e4a9acce86333e6b427ba1651f\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test_view", + "platform": "urn:li:dataPlatform:hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"view\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,hive.db1.array_struct_test_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"select `array_struct_test`.`property_id`, `array_struct_test`.`service` from `db1`.`array_struct_test`\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json new file mode 100644 index 00000000000000..f0915dcdf26cf7 --- /dev/null +++ b/metadata-ingestion/tests/integration/presto-on-hive/presto_on_hive_mces_golden_4.json @@ -0,0 +1,1191 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\"}, \"name\": \"hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e998a77f6edaa92d1326dec9d37c96ab", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Catalog\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"presto-on-hive\", \"instance\": \"PROD\", \"database\": \"hive\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:presto-on-hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bb66ab4651750f727700446f9b3aa2df", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:e998a77f6edaa92d1326dec9d37c96ab\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.map_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].keyvalue", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=map].[type=string].recordid", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.MapType": { + "keyType": "string", + "valueType": "string" + } + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"map\", \"key_type\": {\"type\": \"int\", \"native_data_type\": \"int\", \"_nullable\": true}, \"key_native_data_type\": \"int\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test" + }, + "name": "map_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.map_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.union_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=union].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "union", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=double].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=array].[type=string].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.UnionType": {} + } + }, + "nativeDataType": "struct0", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=int].a", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=union].[type=struct0].foo.[type=string].b", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test" + }, + "name": "union_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.union_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.nested_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=string].name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "varchar(50)", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"varchar(50)\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=struct].provider.[type=int].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"tinyint\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test" + }, + "name": "nested_struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.nested_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "description": "id of property", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service", + "nullable": true, + "description": "service types and providers", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test" + }, + "name": "array_struct_test", + "description": "This table has array of structs", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.struct_test", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=string].type", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].service.[type=array].[type=int].provider", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "int" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test" + }, + "name": "struct_test", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.struct_test,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1._test_table_underscore", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore" + }, + "name": "_test_table_underscore", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1._test_table_underscore,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.pokes", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].foo", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].baz", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].bar", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "create_date": "2022-10-08", + "table_type": "MANAGED_TABLE", + "table_location": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", + "partitioned_columns": "baz" + }, + "name": "pokes", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.pokes,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Table\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bb66ab4651750f727700446f9b3aa2df\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "hive.db1.array_struct_test_presto_view", + "platform": "urn:li:dataPlatform:presto-on-hive", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].property_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=null].service", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "array(row(\"type\" varchar,\"provider\" array(integer)))", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array(row(\\\"type\\\" varchar,\\\"provider\\\" array(integer)))\", \"_nullable\": true}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "is_view": "True" + }, + "name": "array_struct_test_presto_view", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT *\nFROM\n db1.array_struct_test\n", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"View\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:presto-on-hive,hive.db1.array_struct_test_presto_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "value": "{\"materialized\": false, \"viewLogic\": \"SELECT *\\nFROM\\n db1.array_struct_test\\n\", \"viewLanguage\": \"SQL\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "presto-on-hive-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index 315f49a10b218f..c6be2fc1d9a179 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -57,6 +57,8 @@ def loaded_presto_on_hive(presto_on_hive_runner): [ ("hive", False, False, "_1"), ("presto-on-hive", True, True, "_2"), + ("hive", False, False, "_3"), + ("presto-on-hive", True, True, "_4"), ], ) def test_presto_on_hive_ingest( @@ -68,6 +70,7 @@ def test_presto_on_hive_ingest( mode, use_catalog_subtype, use_dataset_pascalcase_subtype, + add_dataset_to_urns, test_suffix, ): @@ -91,6 +94,7 @@ def test_presto_on_hive_ingest( "scheme": "postgresql+psycopg2", "include_views": True, "include_tables": True, + "add_dataset_to_urns": add_dataset_to_urns, "schema_pattern": {"allow": ["^public"]}, "mode": mode, "use_catalog_subtype": use_catalog_subtype, From 1f252627f1eae6df2b6f1213a4dee06a70c726b9 Mon Sep 17 00:00:00 2001 From: treff7es Date: Tue, 18 Oct 2022 22:52:13 +0200 Subject: [PATCH 3/4] Fixing test --- .../presto-on-hive/test_presto_on_hive.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index c6be2fc1d9a179..a80063e09d1db1 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -53,12 +53,12 @@ def loaded_presto_on_hive(presto_on_hive_runner): @freeze_time(FROZEN_TIME) @pytest.mark.integration_batch_1 @pytest.mark.parametrize( - "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,test_suffix", + "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,add_db_name_to_urns,test_suffix", [ - ("hive", False, False, "_1"), - ("presto-on-hive", True, True, "_2"), - ("hive", False, False, "_3"), - ("presto-on-hive", True, True, "_4"), + ("hive", False, False, False, "_1"), + ("presto-on-hive", True, True, False, "_2"), + ("hive", False, False, True, "_3"), + ("presto-on-hive", True, True, True, "_4"), ], ) def test_presto_on_hive_ingest( @@ -70,7 +70,7 @@ def test_presto_on_hive_ingest( mode, use_catalog_subtype, use_dataset_pascalcase_subtype, - add_dataset_to_urns, + add_db_name_to_urns, test_suffix, ): @@ -94,7 +94,7 @@ def test_presto_on_hive_ingest( "scheme": "postgresql+psycopg2", "include_views": True, "include_tables": True, - "add_dataset_to_urns": add_dataset_to_urns, + "add_db_name_to_urns": add_db_name_to_urns, "schema_pattern": {"allow": ["^public"]}, "mode": mode, "use_catalog_subtype": use_catalog_subtype, From cc6932b8b42e4694141605eb38aa0e75078a0a19 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Tue, 18 Oct 2022 17:40:33 -0700 Subject: [PATCH 4/4] changing config name to something more meaningful --- .../src/datahub/ingestion/source/sql/presto_on_hive.py | 10 +++++----- .../integration/presto-on-hive/test_presto_on_hive.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py index c0389a23768cff..d9274246a5a347 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/presto_on_hive.py @@ -121,9 +121,9 @@ class PrestoOnHiveConfig(BasicSQLAlchemyConfig): description="Dataset Subtype name to be 'Table' or 'View' Valid options: ['True', 'False']", ) - add_db_name_to_urns: bool = Field( + include_catalog_name_in_ids: bool = Field( default=False, - description="Add database name/alias to the generated dataset urns", + description="Add the Presto catalog name (e.g. hive) to the generated dataset urns. `urn:li:dataset:(urn:li:dataPlatform:hive,hive.user.logging_events,PROD)` versus `urn:li:dataset:(urn:li:dataPlatform:hive,user.logging_events,PROD)`", ) def get_sql_alchemy_url(self, uri_opts: Optional[Dict[str, Any]] = None) -> str: @@ -415,7 +415,7 @@ def loop_tables( db_name = self.get_db_name(inspector) schema_name = ( f"{db_name}.{key.schema}" - if self.config.add_db_name_to_urns + if self.config.include_catalog_name_in_ids else key.schema ) @@ -537,7 +537,7 @@ def get_hive_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset]: db_name = self.get_db_name(inspector) schema_name = ( f"{db_name}.{key.schema}" - if self.config.add_db_name_to_urns + if self.config.include_catalog_name_in_ids else key.schema ) dataset_name = self.get_identifier( @@ -575,7 +575,7 @@ def get_presto_view_columns(self, inspector: Inspector) -> Iterable[ViewDataset] db_name = self.get_db_name(inspector) schema_name = ( f"{db_name}.{row['schema']}" - if self.config.add_db_name_to_urns + if self.config.include_catalog_name_in_ids else row["schema"] ) dataset_name = self.get_identifier( diff --git a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py index a80063e09d1db1..e4f904ab91e74d 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py +++ b/metadata-ingestion/tests/integration/presto-on-hive/test_presto_on_hive.py @@ -53,7 +53,7 @@ def loaded_presto_on_hive(presto_on_hive_runner): @freeze_time(FROZEN_TIME) @pytest.mark.integration_batch_1 @pytest.mark.parametrize( - "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,add_db_name_to_urns,test_suffix", + "mode,use_catalog_subtype,use_dataset_pascalcase_subtype,include_catalog_name_in_ids,test_suffix", [ ("hive", False, False, False, "_1"), ("presto-on-hive", True, True, False, "_2"), @@ -70,7 +70,7 @@ def test_presto_on_hive_ingest( mode, use_catalog_subtype, use_dataset_pascalcase_subtype, - add_db_name_to_urns, + include_catalog_name_in_ids, test_suffix, ): @@ -94,7 +94,7 @@ def test_presto_on_hive_ingest( "scheme": "postgresql+psycopg2", "include_views": True, "include_tables": True, - "add_db_name_to_urns": add_db_name_to_urns, + "include_catalog_name_in_ids": include_catalog_name_in_ids, "schema_pattern": {"allow": ["^public"]}, "mode": mode, "use_catalog_subtype": use_catalog_subtype,