From aabdd6c4778a4e8ecf2bb4b060e662d17d92042e Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 23 Mar 2023 12:56:12 -0700 Subject: [PATCH] fix(ingest/dbt): enable incremental lineage by default This reverts https://github.com/datahub-project/datahub/pull/6467. --- docs-website/.gitignore | 1 + .../src/datahub/ingestion/source/dbt/dbt_common.py | 5 ----- metadata-ingestion/tests/integration/dbt/test_dbt.py | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docs-website/.gitignore b/docs-website/.gitignore index 9ffda961bfcf0..482a7644292ae 100644 --- a/docs-website/.gitignore +++ b/docs-website/.gitignore @@ -2,6 +2,7 @@ /docs /genDocs +/genStatic # Generated GraphQL /graphql/combined.graphql diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 7e3ce384cd708..afdb1cfba47fe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -267,11 +267,6 @@ class DBTCommonConfig( description='Regex string to extract owner from the dbt node using the `(?P...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P(.*)): (\\w+) (\\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P(.*))"` will extract `alice` as the owner from `"@alice"`.', ) - incremental_lineage: bool = Field( - # Copied from LineageConfig, and changed the default. - default=False, - description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", - ) include_env_in_assertion_guid: bool = Field( default=False, description="Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion " diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index d192026cdef04..a60f4da234d49 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -47,8 +47,6 @@ def set_paths( self.output_path = f"{tmp_path}/{self.output_file}" self.golden_path = f"{test_resources_dir}/{self.golden_file}" - - self.source_config_modifiers.setdefault("incremental_lineage", True) self.source_config = dict( { "manifest_path": self.manifest_path, @@ -254,7 +252,6 @@ def test_dbt_tests(pytestconfig, tmp_path, mock_time, **kwargs): ), # this is just here to avoid needing to access datahub server write_semantics="OVERRIDE", - incremental_lineage=True, ), ), sink=DynamicTypedConfig(type="file", config={"filename": str(output_file)}),