From 38e27937051887d7a3af58c6e63bbac5108ba6c5 Mon Sep 17 00:00:00 2001 From: Onkar Ravgan Date: Fri, 7 Jun 2024 17:59:45 +0530 Subject: [PATCH] MINOR: Enabled pbit file test (#16531) * Enabled pbit file test * Updated test * update test * fixed pylint --- .../powerbi/test_powerbi_file_client.py | 263 ++++-------------- 1 file changed, 48 insertions(+), 215 deletions(-) diff --git a/ingestion/tests/integration/powerbi/test_powerbi_file_client.py b/ingestion/tests/integration/powerbi/test_powerbi_file_client.py index e6979dae3bc7..fd2c3aef7fd1 100644 --- a/ingestion/tests/integration/powerbi/test_powerbi_file_client.py +++ b/ingestion/tests/integration/powerbi/test_powerbi_file_client.py @@ -16,8 +16,6 @@ import os from unittest import TestCase -import pytest - from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import ( PowerBIConnection, ) @@ -25,13 +23,6 @@ PowerBiFileClient, _get_datamodel_schema_list, ) -from metadata.ingestion.source.dashboard.powerbi.models import ( - ConnectionFile, - DataModelSchema, - PowerBiColumns, - PowerBiTable, - RemoteArtifacts, -) current_dir = os.getcwd() @@ -50,207 +41,34 @@ }, } -EXPECTED_DATAMODEL_MAPPINGS = [ - DataModelSchema( - tables=[ - PowerBiTable( - name="customers", - columns=[ - PowerBiColumns( - name="customer_id", dataType="int64", columnType=None - ), - PowerBiColumns( - name="first_name", dataType="string", columnType=None - ), - PowerBiColumns( - name="last_name", dataType="string", columnType=None - ), - PowerBiColumns( - name="first_order", dataType="dateTime", columnType=None - ), - PowerBiColumns( - name="most_recent_order", dataType="dateTime", columnType=None - ), - PowerBiColumns( - name="number_of_orders", dataType="int64", columnType=None - ), - PowerBiColumns( - name="customer_lifetime_value", - dataType="int64", - columnType=None, - ), - ], - description=None, - ), - PowerBiTable( - name="DateTableTemplate_ec327b07-b020-4d43-80c5-516ece79b79d", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - PowerBiTable( - name="LocalDateTable_d0a20cd6-9337-4c89-b71b-8a0e355f79d9", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - PowerBiTable( - name="LocalDateTable_0e74d991-95ed-45b7-9817-97a7fb3e183d", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - PowerBiTable( - name="orders", - columns=[ - PowerBiColumns(name="order_id", dataType="int64", columnType=None), - PowerBiColumns( - name="customer_id", dataType="int64", columnType=None - ), - PowerBiColumns( - name="order_date", dataType="dateTime", columnType=None - ), - PowerBiColumns(name="status", dataType="string", columnType=None), - PowerBiColumns( - name="credit_card_amount", dataType="int64", columnType=None - ), - PowerBiColumns( - name="coupon_amount", dataType="int64", columnType=None - ), - PowerBiColumns( - name="bank_transfer_amount", dataType="int64", columnType=None - ), - PowerBiColumns( - name="gift_card_amount", dataType="int64", columnType=None - ), - PowerBiColumns(name="amount", dataType="int64", columnType=None), - ], - description=None, - ), - PowerBiTable( - name="LocalDateTable_26b94053-21b8-495a-b4f8-33287bd51f89", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - ], - connectionFile=ConnectionFile( - RemoteArtifacts=[ - RemoteArtifacts( - DatasetId="c5bf4b57-1de4-4c7f-ae3a-b151f36a8260", - ReportId="3c8875f2-f68c-4d9a-bacb-4c4b6bf03a12", - ) - ] - ), - ), - DataModelSchema( - tables=[ - PowerBiTable( - name="customers_model", - columns=[ - PowerBiColumns( - name="customer_id", dataType="int64", columnType=None - ), - PowerBiColumns( - name="first_name", dataType="string", columnType=None - ), - PowerBiColumns( - name="last_name", dataType="string", columnType=None - ), - PowerBiColumns( - name="first_order", dataType="dateTime", columnType=None - ), - PowerBiColumns( - name="most_recent_order", dataType="dateTime", columnType=None - ), - PowerBiColumns( - name="number_of_orders", dataType="int64", columnType=None - ), - PowerBiColumns( - name="customer_lifetime_value", - dataType="int64", - columnType=None, - ), - ], - description=None, - ), - PowerBiTable( - name="DateTableTemplate_7775cd0a-0160-433e-a83f-5f387c1bb939", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - PowerBiTable( - name="LocalDateTable_fbc531ce-8c11-4307-a682-c6acc9d92305", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - PowerBiTable( - name="LocalDateTable_fe68b79b-3717-40a9-bd23-dd8aa62713b5", - columns=[ - PowerBiColumns(name="Date", dataType="dateTime", columnType=None), - PowerBiColumns(name="Year", dataType="int64", columnType=None), - PowerBiColumns(name="MonthNo", dataType="int64", columnType=None), - PowerBiColumns(name="Month", dataType="string", columnType=None), - PowerBiColumns(name="QuarterNo", dataType="int64", columnType=None), - PowerBiColumns(name="Quarter", dataType="string", columnType=None), - PowerBiColumns(name="Day", dataType="int64", columnType=None), - ], - description=None, - ), - ], - connectionFile=ConnectionFile( - RemoteArtifacts=[ - RemoteArtifacts( - DatasetId="a7026844-8de5-4419-b312-3162da41ff41", - ReportId="c9b7a5c2-ffaa-4411-a8e9-9099f584dbe9", - ) - ] - ), - ), +EXPECTED_TABLE_NAMES = ["customers", "orders", "customers_model"] + +EXPECTED_COLUMNS = { + "customer_id", + "first_name", + "number_of_orders", + "first_order", + "last_name", + "order_date", + "credit_card_amount", + "bank_transfer_amount", + "amount", + "gift_card_amount", + "customer_lifetime_value", + "order_id", + "status", + "coupon_amount", + "most_recent_order", +} + +EXPECTED_DATASET_IDS = [ + "c5bf4b57-1de4-4c7f-ae3a-b151f36a8260", + "a7026844-8de5-4419-b312-3162da41ff41", +] + +EXPECTED_REPORT_IDS = [ + "3c8875f2-f68c-4d9a-bacb-4c4b6bf03a12", + "c9b7a5c2-ffaa-4411-a8e9-9099f584dbe9", ] @@ -261,7 +79,6 @@ class PowerBIFileClientTests(TestCase): file_client = PowerBiFileClient(PowerBIConnection(**powerbi_connection_config)) - @pytest.mark.skip(reason="TODO: skip this until test is fixed") def test_parsing_pbit_files(self): """ Test unzipping pbit files from local and extract the datamodels and connections @@ -269,7 +86,23 @@ def test_parsing_pbit_files(self): datamodel_mappings = _get_datamodel_schema_list( path=self.file_client.config.pbitFilesSource.path ) - for _, (expected, original) in enumerate( - zip(EXPECTED_DATAMODEL_MAPPINGS, datamodel_mappings) - ): - self.assertEqual(expected, original) + all_tables = [] + for schema in datamodel_mappings: + # test the table and columns from the pbit file + for table in schema.tables: + if table.name in EXPECTED_TABLE_NAMES: + all_tables.append(table.name) + for column in table.columns: + self.assertIn(column.name, EXPECTED_COLUMNS) + + # test the connection objects from the pbit file + for connection in schema.connectionFile.RemoteArtifacts: + self.assertIn(connection.DatasetId, EXPECTED_DATASET_IDS) + self.assertIn( + connection.ReportId, + EXPECTED_REPORT_IDS, + ) + + EXPECTED_TABLE_NAMES.sort() + all_tables.sort() + self.assertEqual(EXPECTED_TABLE_NAMES, all_tables)