diff --git a/.changes/unreleased/Under the Hood-20230706-175507.yaml b/.changes/unreleased/Under the Hood-20230706-175507.yaml new file mode 100644 index 00000000000..5885bf04bd6 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230706-175507.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add option to specify partial parse file +time: 2023-07-06T17:55:07.525287-07:00 +custom: + Author: ChenyuLInx + Issue: "7911" diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index c1a66666066..5c5ace2490e 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -138,6 +138,7 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: @p.log_path @p.macro_debugging @p.partial_parse +@p.partial_parse_file_path @p.populate_cache @p.print @p.printer_width diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 53275149e6e..eff884173fd 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -239,6 +239,15 @@ default=True, ) +partial_parse_file_path = click.option( + "--partial-parse-file-path", + envvar="DBT_PARTIAL_PARSE_FILE_PATH", + help="Internal flag for path to partial_parse.manifest file.", + default=None, + hidden=True, + type=click.Path(exists=True, dir_okay=False, resolve_path=True), +) + populate_cache = click.option( "--populate-cache/--no-populate-cache", envvar="DBT_POPULATE_CACHE", diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index b98b6fe2fc9..25f4e5d6b3a 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -840,10 +840,13 @@ def skip_partial_parsing_because_of_macros(self): return False def read_manifest_for_partial_parse(self) -> Optional[Manifest]: - if not get_flags().PARTIAL_PARSE: + flags = get_flags() + if not flags.PARTIAL_PARSE: fire_event(PartialParsingNotEnabled()) return None - path = os.path.join(self.root_project.project_target_path, PARTIAL_PARSE_FILE_NAME) + path = flags.PARTIAL_PARSE_FILE_PATH or os.path.join( + self.root_project.project_target_path, PARTIAL_PARSE_FILE_NAME + ) reparse_reason = None diff --git a/tests/unit/test_parse_manifest.py b/tests/unit/test_parse_manifest.py index 5d6a90d78ea..5dc39ab74ed 100644 --- a/tests/unit/test_parse_manifest.py +++ b/tests/unit/test_parse_manifest.py @@ -1,12 +1,16 @@ import unittest from unittest import mock -from unittest.mock import patch +from unittest.mock import patch, MagicMock +from argparse import Namespace from .utils import config_from_parts_or_dicts, normalize from dbt.contracts.files import SourceFile, FileHash, FilePath from dbt.contracts.graph.manifest import Manifest, ManifestStateCheck from dbt.parser import manifest +from dbt.parser.manifest import ManifestLoader +from dbt.config import RuntimeConfig +from dbt.flags import set_from_args class MatchingHash(FileHash): @@ -99,3 +103,21 @@ def _new_file(self, searched, name, match): project_root=normalize(self.root_project_config.project_root), ) return SourceFile(path=path, checksum=checksum) + + +class TestPartialParse(unittest.TestCase): + @patch("dbt.parser.manifest.ManifestLoader.build_manifest_state_check") + @patch("dbt.parser.manifest.os.path.exists") + @patch("dbt.parser.manifest.open") + def test_partial_parse_file_path(self, patched_open, patched_os_exist, patched_state_check): + mock_project = MagicMock(RuntimeConfig) + mock_project.project_target_path = "mock_target_path" + patched_os_exist.return_value = True + set_from_args(Namespace(), {}) + ManifestLoader(mock_project, {}) + # by default we use the project_target_path + patched_open.assert_called_with("mock_target_path/partial_parse.msgpack", "rb") + set_from_args(Namespace(partial_parse_file_path="specified_partial_parse_path"), {}) + ManifestLoader(mock_project, {}) + # if specified in flags, we use the specified path + patched_open.assert_called_with("specified_partial_parse_path", "rb")