Add option to lowercase urns in bigquery

datahub-project · Oct 19, 2022 · f0dc352 · f0dc352
1 parent 6373c43
commit f0dc352
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -18,6 +18,7 @@
     make_dataset_urn_with_platform_instance,
     make_domain_urn,
     make_tag_urn,
+    set_dataset_urn_to_lower,
 )
 from datahub.emitter.mcp_builder import (
     BigQueryDatasetKey,
@@ -185,6 +186,8 @@ def __init__(self, ctx: PipelineContext, config: BigQueryV2Config):
             self.config.sharded_table_pattern
         )
 
+        set_dataset_urn_to_lower(self.config.convert_urns_to_lowercase)
+
         # For database, schema, tables, views, etc
         self.lineage_extractor = BigqueryLineageExtractor(config, self.report)
         self.usage_extractor = BigQueryUsageExtractor(config, self.report)

diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py
@@ -77,6 +77,11 @@ class BigQueryV2Config(BigQueryConfig):
         description="Sql parse view ddl to get lineage.",
     )
 
+    convert_urns_to_lowercase: bool = Field(
+        default=False,
+        description="Convert urns to lowercase.",
+    )
+
     @root_validator(pre=False)
     def profile_default_settings(cls, values: Dict) -> Dict:
         # Extra default SQLAlchemy option for better connection pooling and threading.