From cfcfefc85f181ee16bcc2beffbf5e26d4a243e84 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Tue, 1 Nov 2022 21:51:57 +0530 Subject: [PATCH 1/6] WIP --- .../src/datahub/ingestion/source/powerbi.py | 375 ++++++++++++++++-- 1 file changed, 342 insertions(+), 33 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py index 993e74a76f9ab5..219b854f1303cd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py @@ -46,7 +46,7 @@ OwnerClass, OwnershipClass, OwnershipTypeClass, - StatusClass, + StatusClass, SubTypesClass, ) from datahub.utilities.dedup_list import deduplicate_list @@ -62,6 +62,8 @@ class Constant: PBIAccessToken = "PBIAccessToken" DASHBOARD_LIST = "DASHBOARD_LIST" TILE_LIST = "TILE_LIST" + REPORT_LIST = "REPORT_LIST" + PAGE_BY_REPORT = "PAGE_BY_REPORT" DATASET_GET = "DATASET_GET" REPORT_GET = "REPORT_GET" DATASOURCE_GET = "DATASOURCE_GET" @@ -125,6 +127,10 @@ class PowerBiAPIConfig(EnvBasedSourceConfigBase): extract_ownership: bool = pydantic.Field( default=True, description="Whether ownership should be ingested" ) + # Enable/Disable extracting report information + extract_reports: bool = pydantic.Field( + default=False, description="Whether reports should be ingested" + ) class PowerBiDashboardSourceConfig(PowerBiAPIConfig): @@ -144,9 +150,11 @@ class PowerBiAPI: Constant.DATASET_GET: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/datasets/{DATASET_ID}", Constant.DATASOURCE_GET: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/datasets/{DATASET_ID}/datasources", Constant.REPORT_GET: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports/{REPORT_ID}", + Constant.REPORT_LIST: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports", Constant.SCAN_GET: "{POWERBI_ADMIN_BASE_URL}/workspaces/scanStatus/{SCAN_ID}", Constant.SCAN_RESULT_GET: "{POWERBI_ADMIN_BASE_URL}/workspaces/scanResult/{SCAN_ID}", Constant.SCAN_CREATE: "{POWERBI_ADMIN_BASE_URL}/workspaces/getInfo", + Constant.PAGE_BY_REPORT: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports/{REPORT_ID}/pages", } SCOPE: str = "https://analysis.windows.net/powerbi/api/.default" @@ -229,13 +237,47 @@ def __eq__(self, instance): def __hash__(self): return hash(self.__members()) + @dataclass + class Page: + id: str + displayName: str + order: int + + def get_urn_part(self): + return f"pages.{self.id}" + + @dataclass + class User: + id: str + displayName: str + emailAddress: str + graphId: str + principalType: str + + def get_urn_part(self): + return f"users.{self.id}" + + def __members(self): + return self.id, + + def __eq__(self, instance): + return ( + isinstance(instance, PowerBiAPI.User) + and self.__members() == instance.__members() + ) + + def __hash__(self): + return hash(self.__members()) + @dataclass class Report: id: str name: str webUrl: str embedUrl: str - dataset: Any + dataset: "PowerBiAPI.Dataset" + pages: List["PowerBiAPI.Page"] + users: List["PowerBiAPI.User"] def get_urn_part(self): return f"reports.{self.id}" @@ -258,29 +300,6 @@ class CreatedFrom(Enum): def get_urn_part(self): return f"charts.{self.id}" - @dataclass - class User: - id: str - displayName: str - emailAddress: str - dashboardUserAccessRight: str - graphId: str - principalType: str - - def get_urn_part(self): - return f"users.{self.id}" - - def __members(self): - return (self.id,) - - def __eq__(self, instance): - return ( - isinstance(instance, PowerBiAPI.User) - and self.__members() == instance.__members() - ) - - def __hash__(self): - return hash(self.__members()) @dataclass class Dashboard: @@ -291,8 +310,8 @@ class Dashboard: isReadOnly: Any workspace_id: str workspace_name: str - tiles: List[Any] - users: List[Any] + tiles: List["PowerBiAPI.Tile"] + users: List["PowerBiAPI.User"] def get_urn_part(self): return f"dashboards.{self.id}" @@ -372,7 +391,6 @@ def __get_users(self, workspace_id: str, entity: str, id: str) -> List[User]: id=instance.get("identifier"), displayName=instance.get("displayName"), emailAddress=instance.get("emailAddress"), - dashboardUserAccessRight=instance.get("datasetUserAccessRight"), graphId=instance.get("graphId"), principalType=instance.get("principalType"), ) @@ -381,9 +399,9 @@ def __get_users(self, workspace_id: str, entity: str, id: str) -> List[User]: return users - def __get_report(self, workspace_id: str, report_id: str) -> Any: + def __get_report(self, workspace_id: str, report_id: str) -> "PowerBiAPI.Report": """ - Fetch the dataset from PowerBi for the given dataset identifier + Fetch the report from PowerBi for the given report identifier """ if workspace_id is None or report_id is None: LOGGER.info("Input values are None") @@ -420,6 +438,8 @@ def __get_report(self, workspace_id: str, report_id: str) -> Any: name=response_dict.get("name"), webUrl=response_dict.get("webUrl"), embedUrl=response_dict.get("embedUrl"), + users=[], + pages=[], dataset=self.get_dataset( workspace_id=workspace_id, dataset_id=response_dict.get("datasetId") ), @@ -712,6 +732,88 @@ def new_dataset_or_report(tile_instance: Any) -> dict: return tiles + def get_pages_by_report(self, workspace_id: str, report_id: str) -> List["PowerBiAPI.Page"]: + """ + Fetch the report from PowerBi for the given report identifier + """ + if workspace_id is None or report_id is None: + LOGGER.info("workspace_id or report_id is None") + return [] + + pages_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.PAGE_BY_REPORT] + # Replace place holders + pages_endpoint = pages_endpoint.format( + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, + WORKSPACE_ID=workspace_id, + REPORT_ID=report_id, + ) + # Hit PowerBi + LOGGER.info(f"Request to pages URL={pages_endpoint}") + response = requests.get( + pages_endpoint, + headers={Constant.Authorization: self.get_access_token()}, + ) + + # Check if we got response from PowerBi + if response.status_code != 200: + message: str = "Failed to fetch reports from power-bi for" + LOGGER.warning(message) + LOGGER.warning(f"{Constant.WorkspaceId}={workspace_id}") + raise ConnectionError(message) + + response_dict = response.json() + return [ + PowerBiAPI.Page( + id="{}.{}".format(report_id, raw_instance["name"].replace(" ", "_")), + displayName=raw_instance["name"] if raw_instance.get("displayName") is None else raw_instance.get("displayName"), + order=raw_instance.get("order"), + ) for raw_instance in response_dict["value"] + ] + + def get_reports(self, workspace_id: str) -> List["PowerBiAPI.Report"]: + """ + Fetch the report from PowerBi for the given report identifier + """ + if workspace_id is None: + LOGGER.info("workspace id is None") + LOGGER.info(f"{Constant.WorkspaceId}={workspace_id}") + return [] + + report_list_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.REPORT_LIST] + # Replace place holders + report_list_endpoint = report_list_endpoint.format( + POWERBI_BASE_URL=PowerBiAPI.BASE_URL, + WORKSPACE_ID=workspace_id, + ) + # Hit PowerBi + LOGGER.info(f"Request to report URL={report_list_endpoint}") + response = requests.get( + report_list_endpoint, + headers={Constant.Authorization: self.get_access_token()}, + ) + + # Check if we got response from PowerBi + if response.status_code != 200: + message: str = "Failed to fetch reports from power-bi for" + LOGGER.warning(message) + LOGGER.warning(f"{Constant.WorkspaceId}={workspace_id}") + raise ConnectionError(message) + + response_dict = response.json() + return [ + PowerBiAPI.Report( + id=raw_instance["id"], + name=raw_instance.get("name"), + webUrl=raw_instance.get("webUrl"), + embedUrl=raw_instance.get("embedUrl"), + pages=self.get_pages_by_report(workspace_id=workspace_id, report_id=raw_instance["id"]), + users=self.__get_users(workspace_id=workspace_id, entity="reports", id=raw_instance["id"]), + dataset=self.get_dataset( + workspace_id=workspace_id, dataset_id=raw_instance.get("datasetId") + ), + ) for raw_instance in response_dict["value"] + ] + # flake8: noqa: C901 def get_workspace(self, workspace_id: str) -> Workspace: """ @@ -994,7 +1096,7 @@ def __to_datahub_dataset( ) for table in dataset.tables: - # Create an URN for dataset + # Create URN for dataset ds_urn = builder.make_dataset_urn( platform=self.__config.dataset_type_mapping[dataset.datasource.type], name=f"{dataset.datasource.database}.{table.schema_name}.{table.name}", @@ -1303,11 +1405,11 @@ def to_datahub_work_units( ) # Convert user to CorpUser - user_mcps = self.to_datahub_users(dashboard.users) + user_mcps: List[MetadataChangeProposalWrapper] = self.to_datahub_users(dashboard.users) # Convert tiles to charts ds_mcps, chart_mcps = self.to_datahub_chart(dashboard.tiles) # Lets convert dashboard to datahub dashboard - dashboard_mcps = self.__to_datahub_dashboard(dashboard, chart_mcps, user_mcps) + dashboard_mcps: List[MetadataChangeProposalWrapper] = self.__to_datahub_dashboard(dashboard, chart_mcps, user_mcps) # Now add MCPs in sequence mcps.extend(ds_mcps) @@ -1320,6 +1422,207 @@ def to_datahub_work_units( # Return set of work_unit return deduplicate_list([wu for wu in work_units if wu is not None]) + def __pages_to_chart( + self, pages: List[PowerBiAPI.Page], + ds_mcps: List[MetadataChangeProposalWrapper] + ) -> List[MetadataChangeProposalWrapper]: + + chart_mcps = [] + + # Return empty list if input list is empty + if not pages: + return [] + + LOGGER.info(f"Converting pages(count={len(pages)}) to charts") + + def to_chart_mcps(page: PowerBiAPI.Page) -> List[MetadataChangeProposalWrapper]: + LOGGER.info("Converting page {} to chart".format(page.name)) + # Create an URN for chart + chart_urn = builder.make_chart_urn( + self.__config.platform_name, page.get_urn_part() + ) + + LOGGER.info("{}={}".format(Constant.CHART_URN, chart_urn)) + + ds_input: List[str] = self.to_urn_set(ds_mcps) + + # Create chartInfo mcp + # Set chartUrl only if tile is created from Report + chart_info_instance = ChartInfoClass( + title=page.displayName or "", + lastModified=ChangeAuditStamps(), + inputs=ds_input, + customProperties={"order": page.order}, + ) + + info_mcp = self.new_mcp( + entity_type=Constant.CHART, + entity_urn=chart_urn, + aspect_name=Constant.CHART_INFO, + aspect=chart_info_instance, + ) + + # removed status mcp + status_mcp = self.new_mcp( + entity_type=Constant.CHART, + entity_urn=chart_urn, + aspect_name=Constant.STATUS, + aspect=StatusClass(removed=False), + ) + + # ChartKey status + chart_key_instance = ChartKeyClass( + dashboardTool=self.__config.platform_name, + chartId=Constant.CHART_ID.format(page.id), + ) + + chartkey_mcp = self.new_mcp( + entity_type=Constant.CHART, + entity_urn=chart_urn, + aspect_name=Constant.CHART_KEY, + aspect=chart_key_instance, + ) + + return [info_mcp, status_mcp, chartkey_mcp] + + for page in pages: + if page is None: + continue + # Now convert tile to chart MCP + chart_mcp = to_chart_mcps(page, ds_mcps) + chart_mcps.extend(chart_mcp) + + return chart_mcps + + def __report_to_dashboard(self, report: PowerBiAPI.Report, + chart_mcps: List[MetadataChangeProposalWrapper], + user_mcps: List[MetadataChangeProposalWrapper]) -> List[MetadataChangeProposalWrapper]: + """ + Map PowerBi report to Datahub dashboard + """ + + dashboard_urn = builder.make_dashboard_urn( + self.__config.platform_name, report.get_urn_part() + ) + + chart_urn_list: List[str] = self.to_urn_set(chart_mcps) + user_urn_list: List[str] = self.to_urn_set(user_mcps) + + # DashboardInfo mcp + dashboard_info_cls = DashboardInfoClass( + description=report.name or "", + title=report.name or "", + charts=chart_urn_list, + lastModified=ChangeAuditStamps(), + dashboardUrl=report.webUrl, + ) + + info_mcp = self.new_mcp( + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, + aspect_name=Constant.DASHBOARD_INFO, + aspect=dashboard_info_cls, + ) + + # removed status mcp + removed_status_mcp = self.new_mcp( + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, + aspect_name=Constant.STATUS, + aspect=StatusClass(removed=False), + ) + + # dashboardKey mcp + dashboard_key_cls = DashboardKeyClass( + dashboardTool=self.__config.platform_name, + dashboardId=Constant.DASHBOARD_ID.format(report.id), + ) + + # Dashboard key + dashboard_key_mcp = self.new_mcp( + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, + aspect_name=Constant.DASHBOARD_KEY, + aspect=dashboard_key_cls, + ) + + # Dashboard Ownership + owners = [ + OwnerClass(owner=user_urn, type=OwnershipTypeClass.NONE) + for user_urn in user_urn_list + if user_urn is not None + ] + + owner_mcp = None + if len(owners) > 0: + # Dashboard owner MCP + ownership = OwnershipClass(owners=owners) + owner_mcp = self.new_mcp( + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, + aspect_name=Constant.OWNERSHIP, + aspect=ownership, + ) + + # Dashboard browsePaths + browse_path = BrowsePathsClass( + paths=["/powerbi/{}".format(self.__config.workspace_id)] + ) + browse_path_mcp = self.new_mcp( + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, + aspect_name=Constant.BROWSERPATH, + aspect=browse_path, + ) + + sub_type_mcp = self.new_mcp( + entityType=Constant.DASHBOARD, + entityUrn=dashboard_urn, + aspect_name=SubTypesClass.ASPECT_NAME, + aspect=SubTypesClass( + typeNames=["Report"] + ), + ) + + list_of_mcps = [ + browse_path_mcp, + info_mcp, + removed_status_mcp, + dashboard_key_mcp, + sub_type_mcp + ] + + if owner_mcp is not None: + list_of_mcps.append(owner_mcp) + + return list_of_mcps + + def report_to_datahub_work_units(self, report: PowerBiAPI.Report) -> Iterable[MetadataWorkUnit]: + mcps: List[MetadataChangeProposalWrapper] = [] + + LOGGER.info( + f"Converting dashboard={report.name} to datahub dashboard" + ) + + # Convert user to CorpUser + user_mcps = self.to_datahub_users(report.users) + # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization + ds_mcps = self.__to_datahub_dataset(report.dataset) + ds_mcps, chart_mcps = self.__pages_to_chart(report.pages, ds_mcps) + + # Let's convert report to datahub dashboard + report_mcps = self.__report_to_dashboard(report, chart_mcps, user_mcps) + + # Now add MCPs in sequence + mcps.extend(ds_mcps) + mcps.extend(user_mcps) + mcps.extend(chart_mcps) + mcps.extend(report_mcps) + + # Convert MCP to work_units + work_units = map(self.__to_work_unit, mcps) + return work_units + @dataclass class PowerBiDashboardSourceReport(SourceReport): @@ -1402,5 +1705,11 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # Return workunit to Datahub Ingestion framework yield workunit + if self.source_config.extract_reports: + for report in self.powerbi_client.get_reports(workspace_id=workspace.id): + for work_unit in self.mapper.report_to_datahub_work_units(report): + self.reporter.report_workunit(work_unit) + yield workunits + def get_report(self) -> SourceReport: return self.reporter From b4c2457c9958299c4534cb41eb1542216e2b6471 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Wed, 2 Nov 2022 13:14:58 +0530 Subject: [PATCH 2/6] lint fix and integration test --- .../src/datahub/ingestion/source/powerbi.py | 135 +++-- .../golden_test_disabled_ownership.json | 65 +-- .../powerbi/golden_test_ingest.json | 114 +---- .../powerbi/golden_test_report.json | 478 ++++++++++++++++++ .../tests/integration/powerbi/test_powerbi.py | 95 ++++ 5 files changed, 680 insertions(+), 207 deletions(-) create mode 100644 metadata-ingestion/tests/integration/powerbi/golden_test_report.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py index 219b854f1303cd..41e8d497b79185 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py @@ -46,7 +46,8 @@ OwnerClass, OwnershipClass, OwnershipTypeClass, - StatusClass, SubTypesClass, + StatusClass, + SubTypesClass, ) from datahub.utilities.dedup_list import deduplicate_list @@ -172,7 +173,7 @@ class Workspace: name: str state: str dashboards: List[Any] - datasets: Dict + datasets: Dict[str, "PowerBiAPI.Dataset"] @dataclass class DataSource: @@ -241,6 +242,7 @@ def __hash__(self): class Page: id: str displayName: str + name: str order: int def get_urn_part(self): @@ -258,12 +260,12 @@ def get_urn_part(self): return f"users.{self.id}" def __members(self): - return self.id, + return (self.id,) def __eq__(self, instance): return ( - isinstance(instance, PowerBiAPI.User) - and self.__members() == instance.__members() + isinstance(instance, PowerBiAPI.User) + and self.__members() == instance.__members() ) def __hash__(self): @@ -275,7 +277,8 @@ class Report: name: str webUrl: str embedUrl: str - dataset: "PowerBiAPI.Dataset" + description: str + dataset: Optional["PowerBiAPI.Dataset"] pages: List["PowerBiAPI.Page"] users: List["PowerBiAPI.User"] @@ -293,14 +296,13 @@ class CreatedFrom(Enum): id: str title: str embedUrl: str - dataset: Optional[Any] + dataset: Optional["PowerBiAPI.Dataset"] report: Optional[Any] createdFrom: CreatedFrom def get_urn_part(self): return f"charts.{self.id}" - @dataclass class Dashboard: id: str @@ -438,6 +440,7 @@ def __get_report(self, workspace_id: str, report_id: str) -> "PowerBiAPI.Report" name=response_dict.get("name"), webUrl=response_dict.get("webUrl"), embedUrl=response_dict.get("embedUrl"), + description=response_dict.get("description"), users=[], pages=[], dataset=self.get_dataset( @@ -732,7 +735,9 @@ def new_dataset_or_report(tile_instance: Any) -> dict: return tiles - def get_pages_by_report(self, workspace_id: str, report_id: str) -> List["PowerBiAPI.Page"]: + def get_pages_by_report( + self, workspace_id: str, report_id: str + ) -> List["PowerBiAPI.Page"]: """ Fetch the report from PowerBi for the given report identifier """ @@ -765,25 +770,29 @@ def get_pages_by_report(self, workspace_id: str, report_id: str) -> List["PowerB return [ PowerBiAPI.Page( id="{}.{}".format(report_id, raw_instance["name"].replace(" ", "_")), - displayName=raw_instance["name"] if raw_instance.get("displayName") is None else raw_instance.get("displayName"), + name=raw_instance["name"], + displayName=raw_instance.get("displayName"), order=raw_instance.get("order"), - ) for raw_instance in response_dict["value"] + ) + for raw_instance in response_dict["value"] ] - def get_reports(self, workspace_id: str) -> List["PowerBiAPI.Report"]: + def get_reports( + self, workspace: "PowerBiAPI.Workspace" + ) -> List["PowerBiAPI.Report"]: """ Fetch the report from PowerBi for the given report identifier """ - if workspace_id is None: - LOGGER.info("workspace id is None") - LOGGER.info(f"{Constant.WorkspaceId}={workspace_id}") + if workspace is None: + LOGGER.info("workspace is None") + LOGGER.info(f"{Constant.WorkspaceId}={workspace.id}") return [] report_list_endpoint: str = PowerBiAPI.API_ENDPOINTS[Constant.REPORT_LIST] # Replace place holders report_list_endpoint = report_list_endpoint.format( POWERBI_BASE_URL=PowerBiAPI.BASE_URL, - WORKSPACE_ID=workspace_id, + WORKSPACE_ID=workspace.id, ) # Hit PowerBi LOGGER.info(f"Request to report URL={report_list_endpoint}") @@ -796,24 +805,30 @@ def get_reports(self, workspace_id: str) -> List["PowerBiAPI.Report"]: if response.status_code != 200: message: str = "Failed to fetch reports from power-bi for" LOGGER.warning(message) - LOGGER.warning(f"{Constant.WorkspaceId}={workspace_id}") + LOGGER.warning(f"{Constant.WorkspaceId}={workspace.id}") raise ConnectionError(message) response_dict = response.json() - return [ + reports: List["PowerBiAPI.Report"] = [ PowerBiAPI.Report( id=raw_instance["id"], name=raw_instance.get("name"), webUrl=raw_instance.get("webUrl"), embedUrl=raw_instance.get("embedUrl"), - pages=self.get_pages_by_report(workspace_id=workspace_id, report_id=raw_instance["id"]), - users=self.__get_users(workspace_id=workspace_id, entity="reports", id=raw_instance["id"]), - dataset=self.get_dataset( - workspace_id=workspace_id, dataset_id=raw_instance.get("datasetId") + description=raw_instance.get("description"), + pages=self.get_pages_by_report( + workspace_id=workspace.id, report_id=raw_instance["id"] + ), + users=self.__get_users( + workspace_id=workspace.id, entity="reports", id=raw_instance["id"] ), - ) for raw_instance in response_dict["value"] + dataset=workspace.datasets.get(raw_instance.get("datasetId")), + ) + for raw_instance in response_dict["value"] ] + return reports + # flake8: noqa: C901 def get_workspace(self, workspace_id: str) -> Workspace: """ @@ -1288,7 +1303,7 @@ def chart_custom_properties(dashboard: PowerBiAPI.Dashboard) -> dict: # Dashboard browsePaths browse_path = BrowsePathsClass( - paths=["/powerbi/{}".format(self.__config.workspace_id)] + paths=["/powerbi/{}".format(dashboard.workspace_name)] ) browse_path_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, @@ -1405,11 +1420,15 @@ def to_datahub_work_units( ) # Convert user to CorpUser - user_mcps: List[MetadataChangeProposalWrapper] = self.to_datahub_users(dashboard.users) + user_mcps: List[MetadataChangeProposalWrapper] = self.to_datahub_users( + dashboard.users + ) # Convert tiles to charts ds_mcps, chart_mcps = self.to_datahub_chart(dashboard.tiles) # Lets convert dashboard to datahub dashboard - dashboard_mcps: List[MetadataChangeProposalWrapper] = self.__to_datahub_dashboard(dashboard, chart_mcps, user_mcps) + dashboard_mcps: List[ + MetadataChangeProposalWrapper + ] = self.__to_datahub_dashboard(dashboard, chart_mcps, user_mcps) # Now add MCPs in sequence mcps.extend(ds_mcps) @@ -1423,8 +1442,7 @@ def to_datahub_work_units( return deduplicate_list([wu for wu in work_units if wu is not None]) def __pages_to_chart( - self, pages: List[PowerBiAPI.Page], - ds_mcps: List[MetadataChangeProposalWrapper] + self, pages: List[PowerBiAPI.Page], ds_mcps: List[MetadataChangeProposalWrapper] ) -> List[MetadataChangeProposalWrapper]: chart_mcps = [] @@ -1435,8 +1453,10 @@ def __pages_to_chart( LOGGER.info(f"Converting pages(count={len(pages)}) to charts") - def to_chart_mcps(page: PowerBiAPI.Page) -> List[MetadataChangeProposalWrapper]: - LOGGER.info("Converting page {} to chart".format(page.name)) + def to_chart_mcps( + page: PowerBiAPI.Page, ds_mcps: List[MetadataChangeProposalWrapper] + ) -> List[MetadataChangeProposalWrapper]: + LOGGER.info("Converting page {} to chart".format(page.displayName)) # Create an URN for chart chart_urn = builder.make_chart_urn( self.__config.platform_name, page.get_urn_part() @@ -1449,10 +1469,11 @@ def to_chart_mcps(page: PowerBiAPI.Page) -> List[MetadataChangeProposalWrapper]: # Create chartInfo mcp # Set chartUrl only if tile is created from Report chart_info_instance = ChartInfoClass( - title=page.displayName or "", + title=page.name or "", + description=page.displayName or "", lastModified=ChangeAuditStamps(), inputs=ds_input, - customProperties={"order": page.order}, + customProperties={"order": str(page.order)}, ) info_mcp = self.new_mcp( @@ -1483,7 +1504,7 @@ def to_chart_mcps(page: PowerBiAPI.Page) -> List[MetadataChangeProposalWrapper]: aspect=chart_key_instance, ) - return [info_mcp, status_mcp, chartkey_mcp] + return [info_mcp, status_mcp] for page in pages: if page is None: @@ -1494,9 +1515,13 @@ def to_chart_mcps(page: PowerBiAPI.Page) -> List[MetadataChangeProposalWrapper]: return chart_mcps - def __report_to_dashboard(self, report: PowerBiAPI.Report, - chart_mcps: List[MetadataChangeProposalWrapper], - user_mcps: List[MetadataChangeProposalWrapper]) -> List[MetadataChangeProposalWrapper]: + def __report_to_dashboard( + self, + workspace_name: str, + report: PowerBiAPI.Report, + chart_mcps: List[MetadataChangeProposalWrapper], + user_mcps: List[MetadataChangeProposalWrapper], + ) -> List[MetadataChangeProposalWrapper]: """ Map PowerBi report to Datahub dashboard """ @@ -1510,7 +1535,7 @@ def __report_to_dashboard(self, report: PowerBiAPI.Report, # DashboardInfo mcp dashboard_info_cls = DashboardInfoClass( - description=report.name or "", + description=report.description or "", title=report.name or "", charts=chart_urn_list, lastModified=ChangeAuditStamps(), @@ -1565,9 +1590,7 @@ def __report_to_dashboard(self, report: PowerBiAPI.Report, ) # Dashboard browsePaths - browse_path = BrowsePathsClass( - paths=["/powerbi/{}".format(self.__config.workspace_id)] - ) + browse_path = BrowsePathsClass(paths=["/powerbi/{}".format(workspace_name)]) browse_path_mcp = self.new_mcp( entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, @@ -1576,12 +1599,10 @@ def __report_to_dashboard(self, report: PowerBiAPI.Report, ) sub_type_mcp = self.new_mcp( - entityType=Constant.DASHBOARD, - entityUrn=dashboard_urn, + entity_type=Constant.DASHBOARD, + entity_urn=dashboard_urn, aspect_name=SubTypesClass.ASPECT_NAME, - aspect=SubTypesClass( - typeNames=["Report"] - ), + aspect=SubTypesClass(typeNames=["Report"]), ) list_of_mcps = [ @@ -1589,7 +1610,7 @@ def __report_to_dashboard(self, report: PowerBiAPI.Report, info_mcp, removed_status_mcp, dashboard_key_mcp, - sub_type_mcp + sub_type_mcp, ] if owner_mcp is not None: @@ -1597,21 +1618,23 @@ def __report_to_dashboard(self, report: PowerBiAPI.Report, return list_of_mcps - def report_to_datahub_work_units(self, report: PowerBiAPI.Report) -> Iterable[MetadataWorkUnit]: + def report_to_datahub_work_units( + self, report: PowerBiAPI.Report, workspace: PowerBiAPI.Workspace + ) -> Iterable[MetadataWorkUnit]: mcps: List[MetadataChangeProposalWrapper] = [] - LOGGER.info( - f"Converting dashboard={report.name} to datahub dashboard" - ) + LOGGER.info(f"Converting dashboard={report.name} to datahub dashboard") # Convert user to CorpUser user_mcps = self.to_datahub_users(report.users) # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization ds_mcps = self.__to_datahub_dataset(report.dataset) - ds_mcps, chart_mcps = self.__pages_to_chart(report.pages, ds_mcps) + chart_mcps = self.__pages_to_chart(report.pages, ds_mcps) # Let's convert report to datahub dashboard - report_mcps = self.__report_to_dashboard(report, chart_mcps, user_mcps) + report_mcps = self.__report_to_dashboard( + workspace.name, report, chart_mcps, user_mcps + ) # Now add MCPs in sequence mcps.extend(ds_mcps) @@ -1706,10 +1729,12 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: yield workunit if self.source_config.extract_reports: - for report in self.powerbi_client.get_reports(workspace_id=workspace.id): - for work_unit in self.mapper.report_to_datahub_work_units(report): + for report in self.powerbi_client.get_reports(workspace=workspace): + for work_unit in self.mapper.report_to_datahub_work_units( + report, workspace + ): self.reporter.report_workunit(work_unit) - yield workunits + yield work_unit def get_report(self) -> SourceReport: return self.reporter diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index 1d5fde3ec0a713..10aa0d3295e664 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -12,17 +10,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -31,17 +24,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "chartInfo", "aspect": { @@ -50,17 +38,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -69,17 +52,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "chartKey", "aspect": { @@ -88,36 +66,26 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { - "value": "{\"paths\": [\"/powerbi/64ED5CAD-7C10-4684-8180-826122881108\"]}", + "value": "{\"paths\": [\"/powerbi/demo-workspace\"]}", "contentType": "application/json" }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dashboardInfo", "aspect": { @@ -126,17 +94,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -145,17 +108,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dashboardKey", "aspect": { @@ -164,10 +122,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 62ec08afd2f9a2..49bdb95b086022 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -1,9 +1,7 @@ [ { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -12,17 +10,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -31,17 +24,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "corpUserInfo", "aspect": { @@ -50,17 +38,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -69,17 +52,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "corpUserKey", "aspect": { @@ -88,17 +66,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "corpUserInfo", "aspect": { @@ -107,17 +80,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -126,17 +94,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User2@foo.com", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "corpUserKey", "aspect": { @@ -145,17 +108,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "chartInfo", "aspect": { @@ -164,17 +122,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -183,17 +136,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "chartKey", "aspect": { @@ -202,36 +150,26 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "browsePaths", "aspect": { - "value": "{\"paths\": [\"/powerbi/64ED5CAD-7C10-4684-8180-826122881108\"]}", + "value": "{\"paths\": [\"/powerbi/demo-workspace\"]}", "contentType": "application/json" }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dashboardInfo", "aspect": { @@ -240,17 +178,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -259,17 +192,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dashboardKey", "aspect": { @@ -278,17 +206,12 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } }, { - "auditHeader": null, "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -297,10 +220,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "powerbi-test" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json new file mode 100644 index 00000000000000..20b51df7734a60 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -0,0 +1,478 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"description\": \"issue_history\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "value": "{\"active\": true, \"displayName\": \"user1\", \"email\": \"User1@foo.com\", \"title\": \"user1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "value": "{\"username\": \"User1@foo.com\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "value": "{\"active\": true, \"displayName\": \"user2\", \"email\": \"User2@foo.com\", \"title\": \"user2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "value": "{\"username\": \"User2@foo.com\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "value": "{\"customProperties\": {\"datasetId\": \"05169CD2-E713-41E6-9600-1D8066D95445\", \"reportId\": \"\", \"datasetWebUrl\": \"http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details\", \"createdFrom\": \"Dataset\"}, \"title\": \"test_tile\", \"description\": \"test_tile\", \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"inputs\": [{\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", + "changeType": "UPSERT", + "aspectName": "chartKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"chartId\": \"powerbi.linkedin.com/charts/B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "value": "{\"paths\": [\"/powerbi/demo-workspace\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "value": "{\"customProperties\": {\"chartCount\": \"1\", \"workspaceName\": \"demo-workspace\", \"workspaceId\": \"7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}, \"title\": \"test_dashboard\", \"description\": \"test_dashboard\", \"charts\": [\"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)\"], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://localhost/dashboards/web/1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:users.User1@foo.com\", \"type\": \"NONE\"}, {\"owner\": \"urn:li:corpuser:users.User2@foo.com\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "value": "{\"customProperties\": {}, \"description\": \"issue_history\", \"tags\": []}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "value": "{\"active\": true, \"displayName\": \"user1\", \"email\": \"User1@foo.com\", \"title\": \"user1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "value": "{\"username\": \"User1@foo.com\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserInfo", + "aspect": { + "value": "{\"active\": true, \"displayName\": \"user2\", \"email\": \"User2@foo.com\", \"title\": \"user2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "value": "{\"username\": \"User2@foo.com\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "value": "{\"customProperties\": {\"order\": \"0\"}, \"title\": \"ReportSection\", \"description\": \"Regional Sales Analysis\", \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"inputs\": [{\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "value": "{\"customProperties\": {\"order\": \"1\"}, \"title\": \"ReportSection1\", \"description\": \"Geographic Analysis\", \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"inputs\": [{\"string\": \"urn:li:dataset:(urn:li:dataPlatform:postgres,library_db.public.issue_history,DEV)\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "value": "{\"paths\": [\"/powerbi/demo-workspace\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "value": "{\"customProperties\": {}, \"title\": \"SalesMarketing\", \"description\": \"Acryl sales marketing report\", \"charts\": [\"urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)\", \"urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)\"], \"datasets\": [], \"lastModified\": {\"created\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}, \"dashboardUrl\": \"https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "value": "{\"dashboardTool\": \"powerbi\", \"dashboardId\": \"powerbi.linkedin.com/dashboards/5b218778-e7a5-4d73-8187-f10824047715\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Report\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:users.User1@foo.com\", \"type\": \"NONE\"}, {\"owner\": \"urn:li:corpuser:users.User2@foo.com\", \"type\": \"NONE\"}], \"lastModified\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index bbd60f856bd964..147cd4a2c3e8da 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -35,6 +35,30 @@ def register_mock_api(request_mock): ] }, }, + "https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "identifier": "User1@foo.com", + "displayName": "user1", + "emailAddress": "User1@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46", + "principalType": "User", + }, + { + "identifier": "User2@foo.com", + "displayName": "user2", + "emailAddress": "User2@foo.com", + "datasetUserAccessRight": "ReadWrite", + "graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS", + "principalType": "User", + }, + ] + }, + }, "https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": { "method": "GET", "status_code": 200, @@ -140,6 +164,40 @@ def register_mock_api(request_mock): "id": "4674efd1-603c-4129-8d82-03cf2be05aff", }, }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "5b218778-e7a5-4d73-8187-f10824047715", + "name": "SalesMarketing", + "description": "Acryl sales marketing report", + "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", + "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", + } + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "displayName": "Regional Sales Analysis", + "name": "ReportSection", + "order": "0", + }, + { + "displayName": "Geographic Analysis", + "name": "ReportSection1", + "order": "1", + }, + ] + }, + }, } for url in api_vs_response.keys(): @@ -238,3 +296,40 @@ def test_override_ownership( output_path=tmp_path / "powerbi_mces_disabled_ownership.json", golden_path=f"{test_resources_dir}/{mce_out_file}", ) + + +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +def test_extract_reports(mock_msal, pytestconfig, tmp_path, mock_time, requests_mock): + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api(request_mock=requests_mock) + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_source_config(), + "extract_reports": True, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_report_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + mce_out_file = "golden_test_report.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / "powerbi_report_mces.json", + golden_path=f"{test_resources_dir}/{mce_out_file}", + ) From 47f0860e6380cd0208952dd106d9ed28b6fe32e9 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Wed, 2 Nov 2022 13:45:21 +0530 Subject: [PATCH 3/6] doc updated --- .../docs/sources/powerbi/powerbi.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi.md b/metadata-ingestion/docs/sources/powerbi/powerbi.md index 2ef51dbd047cef..c87435a0779685 100644 --- a/metadata-ingestion/docs/sources/powerbi/powerbi.md +++ b/metadata-ingestion/docs/sources/powerbi/powerbi.md @@ -7,13 +7,14 @@ See the - Enhance admin APIs responses with detailed metadata ## Concept mapping -| Power BI | Datahub | -| ------------------------- | ------------------- | -| `Dashboard` | `Dashboard` | -| `Dataset, Datasource` | `Dataset` | -| `Tile` | `Chart` | -| `Report.webUrl` | `Chart.externalUrl` | -| `Workspace` | `N/A` | -| `Report` | `N/A` | +| Power BI | Datahub | +|-----------------------|---------------------| +| `Dashboard` | `Dashboard` | +| `Dataset, Datasource` | `Dataset` | +| `Tile` | `Chart` | +| `Report.webUrl` | `Chart.externalUrl` | +| `Workspace` | `N/A` | +| `Report` | `Dashboard` | +| `Page` | `Chart` | If Tile is created from report then Chart.externalUrl is set to Report.webUrl. From 681da4a7309ca689a759d32857285616b2ed1afb Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Thu, 3 Nov 2022 21:20:50 +0530 Subject: [PATCH 4/6] extract_reports default to true --- metadata-ingestion/src/datahub/ingestion/source/powerbi.py | 2 +- metadata-ingestion/tests/integration/powerbi/test_powerbi.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py index 41e8d497b79185..8a53b3a5e59586 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py @@ -130,7 +130,7 @@ class PowerBiAPIConfig(EnvBasedSourceConfigBase): ) # Enable/Disable extracting report information extract_reports: bool = pydantic.Field( - default=False, description="Whether reports should be ingested" + default=True, description="Whether reports should be ingested" ) diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 147cd4a2c3e8da..d00f4caa5dd777 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -237,6 +237,7 @@ def test_powerbi_ingest(mock_msal, pytestconfig, tmp_path, mock_time, requests_m "type": "powerbi", "config": { **default_source_config(), + "extract_reports": False, }, }, "sink": { @@ -276,6 +277,8 @@ def test_override_ownership( "config": { **default_source_config(), "extract_ownership": False, + "extract_reports": False, + }, }, "sink": { @@ -312,7 +315,6 @@ def test_extract_reports(mock_msal, pytestconfig, tmp_path, mock_time, requests_ "type": "powerbi", "config": { **default_source_config(), - "extract_reports": True, }, }, "sink": { From ea1c4e3de36a79de4f9e0f4138e163c5b6a524ef Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Tue, 15 Nov 2022 08:24:03 +0530 Subject: [PATCH 5/6] review comments --- .../src/datahub/ingestion/source/powerbi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py index 8a53b3a5e59586..4d3c3cdf049a9a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi.py @@ -1451,18 +1451,18 @@ def __pages_to_chart( if not pages: return [] - LOGGER.info(f"Converting pages(count={len(pages)}) to charts") + LOGGER.debug(f"Converting pages(count={len(pages)}) to charts") def to_chart_mcps( page: PowerBiAPI.Page, ds_mcps: List[MetadataChangeProposalWrapper] ) -> List[MetadataChangeProposalWrapper]: - LOGGER.info("Converting page {} to chart".format(page.displayName)) - # Create an URN for chart + LOGGER.debug("Converting page {} to chart".format(page.displayName)) + # Create a URN for chart chart_urn = builder.make_chart_urn( self.__config.platform_name, page.get_urn_part() ) - LOGGER.info("{}={}".format(Constant.CHART_URN, chart_urn)) + LOGGER.debug("{}={}".format(Constant.CHART_URN, chart_urn)) ds_input: List[str] = self.to_urn_set(ds_mcps) @@ -1623,7 +1623,7 @@ def report_to_datahub_work_units( ) -> Iterable[MetadataWorkUnit]: mcps: List[MetadataChangeProposalWrapper] = [] - LOGGER.info(f"Converting dashboard={report.name} to datahub dashboard") + LOGGER.debug(f"Converting dashboard={report.name} to datahub dashboard") # Convert user to CorpUser user_mcps = self.to_datahub_users(report.users) From 61e883112b63a7a0b4ca9fe301ba9de4458e56d6 Mon Sep 17 00:00:00 2001 From: MohdSiddique Bagwan Date: Tue, 15 Nov 2022 12:34:22 +0530 Subject: [PATCH 6/6] lint fix --- metadata-ingestion/tests/integration/powerbi/test_powerbi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index d00f4caa5dd777..a8cb4a60412644 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -278,7 +278,6 @@ def test_override_ownership( **default_source_config(), "extract_ownership": False, "extract_reports": False, - }, }, "sink": {