fix(ingest/bigquery): ignore complex types from profiling (#6613)

datahub-project · Dec 2, 2022 · 43775ec · 43775ec
1 parent 779d1cf
commit 43775ec
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
@@ -165,6 +165,16 @@ def get_workunits(
                     continue
 
                 for table in tables[project][dataset]:
+                    for column in table.columns:
+                        # Profiler has issues with complex types (array, struct, geography, json), so we deny those types from profiling
+                        # We also filter columns without data type as it means that column is part of a complex type.
+                        if not column.data_type or any(
+                            word in column.data_type.lower()
+                            for word in ["array", "struct", "geography", "json"]
+                        ):
+                            self.config.profile_pattern.deny.append(
+                                f"^{project}.{dataset}.{table.name}.{column.field_path}$"
+                            )
                     # Emit the profile work unit
                     profile_request = self.get_bigquery_profile_request(
                         project=project, dataset=dataset, table=table