Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make seeds ref'able, provide for seed configuration #668

Merged
merged 3 commits into from
Feb 28, 2018
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dbt/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,8 @@ def load_project(cls, root_project, all_projects, project, project_name,
all_projects=all_projects,
root_dir=project.get('project-root'),
relative_dirs=project.get('data-paths', []),
resource_type=NodeType.Seed)
resource_type=NodeType.Seed,
macros=macros)


# node loaders
Expand Down
9 changes: 7 additions & 2 deletions dbt/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from dbt.compat import basestring

from dbt.utils import split_path, deep_merge, DBTConfigKeys
from dbt.node_types import NodeType


class SourceConfig(object):
Expand All @@ -24,11 +25,12 @@ class SourceConfig(object):
'bind'
]

def __init__(self, active_project, own_project, fqn):
def __init__(self, active_project, own_project, fqn, node_type):
self._config = None
self.active_project = active_project
self.own_project = own_project
self.fqn = fqn
self.node_type = node_type

# the config options defined within the model
self.in_model_config = {}
Expand Down Expand Up @@ -133,7 +135,10 @@ def get_project_config(self, project):
for k in SourceConfig.ExtendDictFields:
config[k] = {}

model_configs = project.get('models')
if self.node_type == NodeType.Seed:
model_configs = project.get('data')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

data is just the default seed data location right? should this be seeds instead (since the other one is models)?

else:
model_configs = project.get('models')

if model_configs is None:
return config
Expand Down
8 changes: 4 additions & 4 deletions dbt/node_runners.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,16 +79,16 @@ def raise_on_first_error(self):
return False

@classmethod
def is_model(cls, node):
return node.get('resource_type') == NodeType.Model
def is_refable(cls, node):
return node.get('resource_type') in [NodeType.Model, NodeType.Seed]

@classmethod
def is_ephemeral(cls, node):
return dbt.utils.get_materialization(node) == 'ephemeral'

@classmethod
def is_ephemeral_model(cls, node):
return cls.is_model(node) and cls.is_ephemeral(node)
return cls.is_refable(node) and cls.is_ephemeral(node)

def safe_run(self, flat_graph, existing):
catchable_errors = (dbt.exceptions.CompilationException,
Expand Down Expand Up @@ -175,7 +175,7 @@ def do_skip(self):
def get_model_schemas(cls, flat_graph):
schemas = set()
for node in flat_graph['nodes'].values():
if cls.is_model(node) and not cls.is_ephemeral(node):
if cls.is_refable(node) and not cls.is_ephemeral(node):
schemas.add(node['schema'])

return schemas
Expand Down
10 changes: 5 additions & 5 deletions dbt/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,15 @@ def resolve_ref(flat_graph, target_model_name, target_model_package,
current_project, node_package):

if target_model_package is not None:
return dbt.utils.find_model_by_name(
return dbt.utils.find_refable_by_name(
flat_graph,
target_model_name,
target_model_package)

target_model = None

# first pass: look for models in the current_project
target_model = dbt.utils.find_model_by_name(
target_model = dbt.utils.find_refable_by_name(
flat_graph,
target_model_name,
current_project)
Expand All @@ -56,7 +56,7 @@ def resolve_ref(flat_graph, target_model_name, target_model_package,
return target_model

# second pass: look for models in the node's package
target_model = dbt.utils.find_model_by_name(
target_model = dbt.utils.find_refable_by_name(
flat_graph,
target_model_name,
node_package)
Expand All @@ -67,7 +67,7 @@ def resolve_ref(flat_graph, target_model_name, target_model_package,
# final pass: look for models in any package
# todo: exclude the packages we have already searched. overriding
# a package model in another package doesn't necessarily work atm
return dbt.utils.find_model_by_name(
return dbt.utils.find_refable_by_name(
flat_graph,
target_model_name,
None)
Expand Down Expand Up @@ -201,7 +201,7 @@ def parse_node(node, node_path, root_project_config, package_project_config,
fqn = get_fqn(node.get('path'), package_project_config, fqn_extra)

config = dbt.model.SourceConfig(
root_project_config, package_project_config, fqn)
root_project_config, package_project_config, fqn, node['resource_type'])

node['unique_id'] = node_path
node['empty'] = ('raw_sql' in node and len(node['raw_sql'].strip()) == 0)
Expand Down
8 changes: 4 additions & 4 deletions dbt/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,14 @@ def model_immediate_name(model, non_destructive):
return "{}__dbt_tmp".format(model_name)


def find_model_by_name(flat_graph, target_name, target_package):
def find_refable_by_name(flat_graph, target_name, target_package):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

return find_by_name(flat_graph, target_name, target_package,
'nodes', NodeType.Model)
'nodes', [NodeType.Model, NodeType.Seed])


def find_macro_by_name(flat_graph, target_name, target_package):
return find_by_name(flat_graph, target_name, target_package,
'macros', NodeType.Macro)
'macros', [NodeType.Macro])


def find_by_name(flat_graph, target_name, target_package, subgraph,
Expand All @@ -146,7 +146,7 @@ def find_by_name(flat_graph, target_name, target_package, subgraph,

resource_type, package_name, node_name = node_parts

if (resource_type == nodetype and
if (resource_type in nodetype and
((target_name == node_name) and
(target_package is None or
target_package == package_name))):
Expand Down