Skip to content

Commit

Permalink
fix(ingest/snowflake): avoid overfetching schemas from datahub (#10527)
Browse files Browse the repository at this point in the history
  • Loading branch information
hsheth2 authored and yoonhyejin committed Jul 16, 2024
1 parent 1d3c4dd commit 09000d4
Showing 1 changed file with 13 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,18 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config):
platform=self.platform,
platform_instance=self.config.platform_instance,
env=self.config.env,
graph=self.ctx.graph,
graph=(
# If we're ingestion schema metadata for tables/views, then we will populate
# schemas into the resolver as we go. We only need to do a bulk fetch
# if we're not ingesting schema metadata as part of ingestion.
self.ctx.graph
if not (
self.config.include_technical_schema
and self.config.include_tables
and self.config.include_views
)
else None
),
generate_usage_statistics=False,
generate_operations=False,
format_queries=self.config.format_sql_queries,
Expand Down Expand Up @@ -1252,7 +1263,7 @@ def gen_schema_metadata(
foreignKeys=foreign_keys,
)

if self.aggregator and self.config.parse_view_ddl:
if self.aggregator:
self.aggregator.register_schema(urn=dataset_urn, schema=schema_metadata)

return schema_metadata
Expand Down

0 comments on commit 09000d4

Please sign in to comment.