Skip to content

Commit

Permalink
fix(ingest): unity - Removing unneeded dependency for sqlalchemy from…
Browse files Browse the repository at this point in the history
… unity-catalog to fix connector setup (datahub-project#6379)
  • Loading branch information
treff7es authored and cccs-Dustin committed Feb 1, 2023
1 parent a9a2ffd commit a206d62
Showing 1 changed file with 22 additions and 2 deletions.
24 changes: 22 additions & 2 deletions metadata-ingestion/src/datahub/ingestion/source/unity/report.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
from dataclasses import dataclass, field
from typing import Dict

from datahub.ingestion.source.sql.sql_common import SQLSourceReport
from datahub.ingestion.source.state.stale_entity_removal_handler import (
StaleEntityRemovalSourceReport,
)
from datahub.utilities.lossy_collections import LossyList
from datahub.utilities.stats_collections import TopKDict


@dataclass
class UnityCatalogReport(SQLSourceReport):
class UnityCatalogReport(StaleEntityRemovalSourceReport):
scanned_metastore: int = 0
scanned_catalog: int = 0
scanned_schema: int = 0
scanned_table: int = 0
num_catalogs_to_scan: Dict[str, int] = field(default_factory=TopKDict)
num_schemas_to_scan: Dict[str, int] = field(default_factory=TopKDict)
num_tables_to_scan: Dict[str, int] = field(default_factory=TopKDict)
tables_scanned: int = 0
views_scanned: int = 0
filtered: LossyList[str] = field(default_factory=LossyList)

def increment_scanned_metastore(self, count: int = 1) -> None:
self.scanned_metastore = self.scanned_metastore + count
Expand All @@ -26,3 +32,17 @@ def increment_scanned_schema(self, count: int = 1) -> None:

def increment_scanned_table(self, count: int = 1) -> None:
self.scanned_table = self.scanned_table + count

def report_dropped(self, ent_name: str) -> None:
self.filtered.append(ent_name)

def report_entity_scanned(self, name: str, ent_type: str = "table") -> None:
"""
Entity could be a view or a table
"""
if ent_type == "table":
self.tables_scanned += 1
elif ent_type == "view":
self.views_scanned += 1
else:
raise KeyError(f"Unknown entity {ent_type}.")

0 comments on commit a206d62

Please sign in to comment.