Skip to content

Commit

Permalink
don't use meta! need better separation between dbt internal objects…
Browse files Browse the repository at this point in the history
… and external facing data. hacked an internal field on the manifest to save off this parsing info for the time being
  • Loading branch information
Kyle Wigley committed Jun 25, 2021
1 parent 3763656 commit a195e28
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 16 deletions.
10 changes: 10 additions & 0 deletions core/dbt/contracts/graph/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,12 @@ def _find_macros_by_name(
return candidates


@dataclass
class ParsingInfo:
static_analysis_parsed_path_count: int = 0
static_analysis_path_count: int = 0


@dataclass
class ManifestStateCheck(dbtClassMixin):
vars_hash: FileHash = field(default_factory=FileHash.empty)
Expand Down Expand Up @@ -566,6 +572,10 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
_analysis_lookup: Optional[AnalysisLookup] = field(
default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
)
_parsing_info: ParsingInfo = field(
default_factory=ParsingInfo,
metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
)
_lock: Lock = field(
default_factory=flags.MP_CONTEXT.Lock,
metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
Expand Down
1 change: 0 additions & 1 deletion core/dbt/contracts/graph/parsed.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,6 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
compiled_path: Optional[str] = None
build_path: Optional[str] = None
deferred: bool = False
unrendered_config: Dict[str, Any] = field(default_factory=dict)
created_at: int = field(default_factory=lambda: int(time.time()))

def write_node(self, target_path: str, subdirectory: str, payload: str):
Expand Down
22 changes: 10 additions & 12 deletions core/dbt/parser/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ class ManifestLoaderInfo(dbtClassMixin, Writable):
static_analysis_path_count: int = 0
static_analysis_parsed_path_count: int = 0
is_partial_parse_enabled: Optional[bool] = None
is_static_analysis_enabled: Optional[bool] = None
read_files_elapsed: Optional[float] = None
load_macros_elapsed: Optional[float] = None
parse_project_elapsed: Optional[float] = None
Expand Down Expand Up @@ -307,16 +308,12 @@ def load(self):
time.perf_counter() - start_process
)
# update tracking data with static parsing info
statically_extracted_nodes = [
node for node in self.manifest.nodes.values()
if node.meta.pop('is_statically_extracted', False) is True
]
statically_extractable_nodes = [
node for node in self.manifest.nodes.values()
if node.meta.pop('is_statically_extractable', False) is True
]
self._perf_info.static_analysis_parsed_path_count = len(statically_extracted_nodes)
self._perf_info.static_analysis_path_count = len(statically_extractable_nodes)
self._perf_info.static_analysis_parsed_path_count = (
self.manifest._parsing_info.static_analysis_parsed_path_count
)
self._perf_info.static_analysis_path_count = (
self.manifest._parsing_info.static_analysis_path_count
)

# write out the fully parsed manifest
self.write_manifest_for_partial_parse()
Expand Down Expand Up @@ -516,7 +513,8 @@ def read_manifest_for_partial_parse(self) -> Optional[Manifest]:

def build_perf_info(self):
mli = ManifestLoaderInfo(
is_partial_parse_enabled=self._partial_parse_enabled()
is_partial_parse_enabled=self._partial_parse_enabled(),
is_static_analysis_enabled=flags.USE_EXPERIMENTAL_PARSER
)
for project in self.all_projects.values():
project_info = ProjectLoaderInfo(
Expand Down Expand Up @@ -629,7 +627,7 @@ def track_project_load(self):
"is_partial_parse_enabled": (
self._perf_info.is_partial_parse_enabled
),
"is_static_analysis_enabled": flags.USE_EXPERIMENTAL_PARSER,
"is_static_analysis_enabled": self._perf_info.is_static_analysis_enabled,
"static_analysis_path_count": self._perf_info.static_analysis_path_count,
"static_analysis_parsed_path_count": self._perf_info.static_analysis_parsed_path_count,
})
Expand Down
4 changes: 2 additions & 2 deletions core/dbt/parser/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def get_compiled_path(cls, block: FileBlock):
def render_update(
self, node: ParsedModelNode, config: ContextConfig
) -> None:
node.meta["is_statically_extractable"] = True
self.manifest._parsing_info.static_analysis_path_count += 1

# normal dbt run
if not flags.USE_EXPERIMENTAL_PARSER:
Expand Down Expand Up @@ -64,7 +64,7 @@ def render_update(
for configv in res['configs']:
node.config[configv[0]] = configv[1]

node.meta['is_statically_extracted'] = True
self.manifest._parsing_info.static_analysis_parsed_path_count += 1

else:
super().render_update(node, config)
1 change: 0 additions & 1 deletion test/unit/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -501,7 +501,6 @@ def test_basic(self):
original_file_path=normalize('models/nested/model_1.sql'),
root_path=get_abs_os_path('./dbt_modules/snowplow'),
config=NodeConfig(materialized='table'),
meta={'is_statically_extractable': True},
path=normalize('nested/model_1.sql'),
raw_sql=raw_sql,
checksum=block.file.checksum,
Expand Down

0 comments on commit a195e28

Please sign in to comment.