-
Notifications
You must be signed in to change notification settings - Fork 1.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Spike] Model versions parsing #7204
Changes from 1 commit
9b9d02d
9b2f8fe
9b5acff
810cec1
96b2a6f
cc3b6af
3d12e45
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,7 +3,7 @@ | |
import pathlib | ||
|
||
from abc import ABCMeta, abstractmethod | ||
from typing import Iterable, Dict, Any, Union, List, Optional, Generic, TypeVar, Type | ||
from typing import Iterable, Dict, Any, Union, List, Optional, Generic, TypeVar, Type, Sequence | ||
|
||
from dbt.dataclass_schema import ValidationError, dbtClassMixin | ||
|
||
|
@@ -53,6 +53,7 @@ | |
UnparsedMetric, | ||
UnparsedSourceDefinition, | ||
UnparsedGroup, | ||
UnparsedVersion, | ||
) | ||
from dbt.exceptions import ( | ||
CompilationError, | ||
|
@@ -151,6 +152,21 @@ def from_target(cls, target: Union[HasColumnDocs, HasColumnTests]) -> "ParserRef | |
refs._add(column) | ||
return refs | ||
|
||
@classmethod | ||
def from_version( | ||
cls, base_columns: Sequence[HasColumnProps], version: UnparsedVersion | ||
) -> "ParserRef": | ||
refs = cls() | ||
for base_column in base_columns: | ||
if version.include_exclude.includes(base_column.name): | ||
refs._add(base_column) | ||
|
||
for column in version.columns: | ||
if isinstance(column, UnparsedColumn): | ||
refs._add(column) | ||
|
||
return refs | ||
|
||
|
||
def _trimmed(inp: str) -> str: | ||
if len(inp) < 50: | ||
|
@@ -890,7 +906,7 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: | |
f"file {source_file.path.original_file_path}" | ||
) | ||
|
||
# patch versioned nodes. | ||
# patch versioned nodes | ||
versions = block.target.versions | ||
if versions: | ||
latest_version = block.target.latest_version or max( | ||
|
@@ -920,13 +936,19 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: | |
versioned_model_node.fqn[-1] = patch.name # bleugh | ||
versioned_model_node.fqn.append(f"v{unparsed_version.name}") | ||
self.manifest.nodes[versioned_model_node.unique_id] = versioned_model_node | ||
|
||
# flatten columns based on include/exclude | ||
version_refs: ParserRef = ParserRef.from_version( | ||
block.target.columns, unparsed_version | ||
) | ||
|
||
versioned_model_patch = ParsedNodePatch( | ||
name=patch.name, | ||
original_file_path=versioned_model_node.original_file_path, | ||
yaml_key=patch.yaml_key, | ||
package_name=versioned_model_node.package_name, | ||
description=unparsed_version.description or versioned_model_node.description, | ||
columns={}, # TODO: flatten columns based on include/exclude | ||
columns=version_refs.column_info, | ||
meta=unparsed_version.meta or versioned_model_node.meta, | ||
docs=unparsed_version.docs or versioned_model_node.docs, | ||
config=unparsed_version.config or versioned_model_node.config, | ||
|
@@ -935,8 +957,8 @@ def parse_patch(self, block: TargetBlock[NodeTarget], refs: ParserRef) -> None: | |
is_latest_version=latest_version == unparsed_version.name, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. opted to have an Could alternatively keep |
||
) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. some of these fields need a closer inspection - but the gist of it is that the versioned model node should inherit properties and configs from the base node patch (where the |
||
versioned_model_node.patch(versioned_model_patch) | ||
# TODO - update alias | ||
self.manifest.rebuild_ref_lookup() # TODO: is this necessary at this point? | ||
# TODO - update alias | ||
return | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this doesn't handle parsing any |
||
|
||
# handle disabled nodes | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is the meat and potatoes of the change. Currently it's in
NonSourceParser
, which is subclassed byTestablePatchParser
for parsing models, seeds, and snapshots. Given the increasing number of model-specific patching that's emerged as part of the 'Models as APIs' work (access, constraints, now versions) - I'm thinking this could be split out into a new subclass for parsing models patches independently of seeds and snapshots. Perhaps simplyModelPatchParser
. What do you think @gshank, cc @peterallenwebb?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've never been enthusiastic about having all that code in common. I understand why it was that way to start with, since originally there were no differences, but it provides friction when we need and want to handle differences. I'd be in favor of having them all subclasses, even if some of them were almost duplicates to start with.