diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/TypeIcon.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/TypeIcon.tsx index c9e7e6de17c6ba..db7fb1319f94c4 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/TypeIcon.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/TypeIcon.tsx @@ -77,6 +77,10 @@ type Props = { export default function TypeIcon({ type, nativeDataType }: Props) { const { icon: Icon, size, text } = DATA_TYPE_ICON_MAP[type]; + + // if unable to match type to DataHub, display native type info by default + const nativeFallback = type === SchemaFieldDataType.Null; + // eslint-disable-next-line react/prop-types const NativeDataTypeTooltip = ({ children }) => nativeDataType ? ( @@ -92,7 +96,7 @@ export default function TypeIcon({ type, nativeDataType }: Props) { {Icon && } - {text} + {nativeFallback ? nativeDataType : text} diff --git a/metadata-ingestion/examples/recipes/dbt_to_datahub.yml b/metadata-ingestion/examples/recipes/dbt_to_datahub.yml index e5823d27b42045..04940b8580a5ba 100644 --- a/metadata-ingestion/examples/recipes/dbt_to_datahub.yml +++ b/metadata-ingestion/examples/recipes/dbt_to_datahub.yml @@ -2,11 +2,11 @@ source: type: "dbt" config: - manifest_path: "./your/dbt/manifest.json" - catalog_path: "./your/dbt/catalog.json" - target_platform: "[target data platform]" - load_schemas: True # or false + manifest_path: "./tests/integration/dbt/dbt_manifest.json" + catalog_path: "./tests/integration/dbt/dbt_catalog.json" + target_platform: "dbt" + load_schemas: True # or False sink: type: "datahub-rest" config: - server: 'http://localhost:8080' + server: "http://localhost:8080" diff --git a/metadata-ingestion/scripts/update_golden_files.sh b/metadata-ingestion/scripts/update_golden_files.sh index ad4e968f932414..3861943c181b13 100755 --- a/metadata-ingestion/scripts/update_golden_files.sh +++ b/metadata-ingestion/scripts/update_golden_files.sh @@ -8,11 +8,12 @@ pytest --basetemp=tmp || true # Update the golden files. cp tmp/test_serde_to_json_tests_unit_0/output.json tests/unit/serde/test_serde_large.json cp tmp/test_serde_to_json_tests_unit_1/output.json tests/unit/serde/test_serde_chart_snapshot.json -cp tmp/test_ldap_ingest0/ldap_mces.json tests/integration/ldap/ldap_mce_golden.json -cp tmp/test_mysql_ingest0/mysql_mces.json tests/integration/mysql/mysql_mce_golden.json -cp tmp/test_mssql_ingest0/mssql_mces.json tests/integration/sql_server/mssql_mce_golden.json -cp tmp/test_mongodb_ingest0/mongodb_mces.json tests/integration/mongodb/mongodb_mce_golden.json -cp tmp/test_feast_ingest0/feast_mces.json tests/integration/feast/feast_mce_golden.json +cp tmp/test_ldap_ingest0/ldap_mces.json tests/integration/ldap/ldap_mces_golden.json +cp tmp/test_mysql_ingest0/mysql_mces.json tests/integration/mysql/mysql_mces_golden.json +cp tmp/test_mssql_ingest0/mssql_mces.json tests/integration/sql_server/mssql_mces_golden.json +cp tmp/test_mongodb_ingest0/mongodb_mces.json tests/integration/mongodb/mongodb_mces_golden.json +cp tmp/test_feast_ingest0/feast_mces.json tests/integration/feast/feast_mces_golden.json +cp tmp/test_dbt_ingest0/dbt_mces.json tests/integration/dbt/dbt_mces_golden.json cp tmp/test_lookml_ingest0/lookml_mces.json tests/integration/lookml/expected_output.json cp tmp/test_looker_ingest0/looker_mces.json tests/integration/looker/expected_output.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index 97199417e45ce9..95a471d2e09342 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -1,6 +1,5 @@ import json import logging -import re import time from typing import Any, Dict, Iterable, List @@ -8,6 +7,11 @@ from datahub.configuration.common import AllowDenyPattern from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.source.dbt_types import ( + POSTGRES_TYPES_MAP, + SNOWFLAKE_TYPES_MAP, + resolve_postgres_modified_type, +) from datahub.ingestion.source.metadata_common import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import AuditStamp from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( @@ -27,6 +31,7 @@ SchemaFieldDataType, SchemaMetadata, StringTypeClass, + TimeTypeClass, ) from datahub.metadata.schema_classes import DatasetPropertiesClass @@ -96,8 +101,7 @@ def extract_dbt_entities( node_type_pattern: AllowDenyPattern, ) -> List[DBTNode]: dbt_entities = [] - for key in nodes: - node = nodes[key] + for key, node in nodes.items(): dbtNode = DBTNode() # check if node pattern allowed based on config file @@ -174,7 +178,7 @@ def loadManifestAndCatalog( all_catalog_entities = {**catalog_nodes, **catalog_sources} - nodes = extract_dbt_entities( + return extract_dbt_entities( all_manifest_entities, all_catalog_entities, load_catalog, @@ -183,8 +187,6 @@ def loadManifestAndCatalog( node_type_pattern, ) - return nodes - def get_urn_from_dbtNode( database: str, schema: str, name: str, target_platform: str, env: str @@ -195,11 +197,11 @@ def get_urn_from_dbtNode( def get_custom_properties(node: DBTNode) -> Dict[str, str]: - properties = {} - properties["dbt_node_type"] = node.node_type - properties["materialization"] = node.materialization - properties["dbt_file_path"] = node.dbt_file_path - return properties + return { + "dbt_node_type": node.node_type, + "materialization": node.materialization, + "dbt_file_path": node.dbt_file_path, + } def get_upstreams( @@ -216,7 +218,7 @@ def get_upstreams( dbtNode_upstream.database = all_nodes[upstream]["database"] dbtNode_upstream.schema = all_nodes[upstream]["schema"] - if "identifier" in all_nodes[upstream] and load_catalog is False: + if "identifier" in all_nodes[upstream] and not load_catalog: dbtNode_upstream.name = all_nodes[upstream]["identifier"] else: dbtNode_upstream.name = all_nodes[upstream]["name"] @@ -247,20 +249,22 @@ def get_upstream_lineage(upstream_urns: List[str]) -> UpstreamLineage: ) ucl.append(uc) - ulc = UpstreamLineage(upstreams=ucl) - - return ulc + return UpstreamLineage(upstreams=ucl) -# This is from a fairly narrow data source that is posgres specific, we would expect this to expand over -# time or be replaced with a more thorough mechanism +# See https://github.com/fishtown-analytics/dbt/blob/master/core/dbt/adapters/sql/impl.py _field_type_mapping = { "boolean": BooleanTypeClass, "date": DateTypeClass, + "time": TimeTypeClass, "numeric": NumberTypeClass, "text": StringTypeClass, "timestamp with time zone": DateTypeClass, + "timestamp without time zone": DateTypeClass, "integer": NumberTypeClass, + "float8": NumberTypeClass, + **POSTGRES_TYPES_MAP, + **SNOWFLAKE_TYPES_MAP, } @@ -270,20 +274,16 @@ def get_column_type( """ Maps known DBT types to datahub types """ - column_type_stripped = "" + TypeClass: Any = _field_type_mapping.get(column_type) - pattern = re.compile(r"[\w ]+") # drop all non alphanumerics - match = pattern.match(column_type) - if match is not None: - column_type_stripped = match.group() + if TypeClass is None: - TypeClass: Any = None - for key in _field_type_mapping.keys(): - if key == column_type_stripped: - TypeClass = _field_type_mapping[column_type_stripped] - break + # attempt Postgres modified type + TypeClass = resolve_postgres_modified_type(column_type) + # if still not found, report the warning if TypeClass is None: + report.report_warning( dataset_name, f"unable to map type {column_type} to metadata schema" ) @@ -309,7 +309,7 @@ def get_schema_metadata( canonical_schema.append(field) actor, sys_time = "urn:li:corpuser:dbt_executor", int(time.time()) * 1000 - schema_metadata = SchemaMetadata( + return SchemaMetadata( schemaName=node.dbt_name, platform=f"urn:li:dataPlatform:{platform}", version=0, @@ -319,7 +319,6 @@ def get_schema_metadata( lastModified=AuditStamp(time=sys_time, actor=actor), fields=canonical_schema, ) - return schema_metadata class DBTSource(Source): diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt_types.py b/metadata-ingestion/src/datahub/ingestion/source/dbt_types.py new file mode 100644 index 00000000000000..554e84b7afd907 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt_types.py @@ -0,0 +1,264 @@ +import re +from typing import Any, Dict, ValuesView + +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + ArrayType, + BooleanType, + BytesType, + DateType, + EnumType, + NullType, + NumberType, + RecordType, + StringType, + TimeType, + UnionType, +) + +# these can be obtained by running `select format_type(oid, null),* from pg_type;` +# we've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.) +# (run `\copy (select format_type(oid, null),* from pg_type) to 'pg_type.csv' csv header;` to get a CSV) + +# we map from format_type since this is what dbt uses +# see https://github.com/fishtown-analytics/dbt/blob/master/plugins/postgres/dbt/include/postgres/macros/catalog.sql#L22 + +# see https://www.npgsql.org/dev/types.html for helpful type annotations +POSTGRES_TYPES_MAP: Dict[str, Any] = { + "boolean": BooleanType, + "bytea": BytesType, + '"char"': StringType, + "name": None, # for system identifiers + "bigint": NumberType, + "smallint": NumberType, + "int2vector": NumberType, # for indexing + "integer": NumberType, + "regproc": None, # object identifier + "text": StringType, + "oid": None, # object identifier + "tid": None, # object identifier + "xid": None, # object identifier + "cid": None, # object identifier + "oidvector": None, # object identifier + "json": RecordType, + "xml": RecordType, + "xid8": None, # object identifier + "point": None, # 2D point + "lseg": None, # line segment + "path": None, # path of points + "box": None, # a pair of corner points + "polygon": None, # closed set of points + "line": None, # infinite line + "real": NumberType, + "double precision": NumberType, + "unknown": None, + "circle": None, # circle with center and radius + "money": NumberType, + "macaddr": None, # MAC address + "inet": None, # IPv4 or IPv6 host address + "cidr": None, # IPv4 or IPv6 network specification + "macaddr8": None, # MAC address + "aclitem": None, # system info + "character": StringType, + "character varying": StringType, + "date": DateType, + "time without time zone": TimeType, + "timestamp without time zone": TimeType, + "timestamp with time zone": TimeType, + "interval": None, + "time with time zone": TimeType, + "bit": BytesType, + "bit varying": BytesType, + "numeric": NumberType, + "refcursor": None, + "regprocedure": None, + "regoper": None, + "regoperator": None, + "regclass": None, + "regcollation": None, + "regtype": None, + "regrole": None, + "regnamespace": None, + "uuid": StringType, + "pg_lsn": None, + "tsvector": None, # text search vector + "gtsvector": None, # GiST for tsvector. Probably internal type. + "tsquery": None, # text search query tree + "regconfig": None, + "regdictionary": None, + "jsonb": BytesType, + "jsonpath": None, # path to property in a JSON doc + "txid_snapshot": None, + "pg_snapshot": None, + "int4range": None, # don't have support for ranges yet + "numrange": None, + "tsrange": None, + "tstzrange": None, + "daterange": None, + "int8range": None, + "record": RecordType, + "record[]": ArrayType, + "cstring": None, + '"any"': UnionType, + "anyarray": ArrayType, + "void": NullType, + "trigger": None, + "event_trigger": None, + "language_handler": None, + "internal": None, + "anyelement": None, + "anynonarray": None, + "anyenum": EnumType, + "fdw_handler": None, + "index_am_handler": None, + "tsm_handler": None, + "table_am_handler": None, + "anyrange": None, + "anycompatible": None, + "anycompatiblearray": None, + "anycompatiblenonarray": None, + "anycompatiblerange": None, + "boolean[]": ArrayType, + "bytea[]": ArrayType, + '"char"[]': ArrayType, + "name[]": ArrayType, + "bigint[]": ArrayType, + "smallint[]": ArrayType, + "int2vector[]": ArrayType, + "integer[]": ArrayType, + "regproc[]": ArrayType, + "text[]": ArrayType, + "oid[]": ArrayType, + "tid[]": ArrayType, + "xid[]": ArrayType, + "cid[]": ArrayType, + "oidvector[]": ArrayType, + "json[]": ArrayType, + "xml[]": ArrayType, + "xid8[]": ValuesView, + "point[]": ArrayType, + "lseg[]": ArrayType, + "path[]": ArrayType, + "box[]": ArrayType, + "polygon[]": ArrayType, + "line[]": ArrayType, + "real[]": ArrayType, + "double precision[]": ArrayType, + "circle[]": ArrayType, + "money[]": ArrayType, + "macaddr[]": ArrayType, + "inet[]": ArrayType, + "cidr[]": ArrayType, + "macaddr8[]": ArrayType, + "aclitem[]": ArrayType, + "character[]": ArrayType, + "character varying[]": ArrayType, + "date[]": ArrayType, + "time without time zone[]": ArrayType, + "timestamp without time zone[]": ArrayType, + "timestamp with time zone[]": ArrayType, + "interval[]": ArrayType, + "time with time zone[]": ArrayType, + "bit[]": ArrayType, + "bit varying[]": ArrayType, + "numeric[]": ArrayType, + "refcursor[]": ArrayType, + "regprocedure[]": ArrayType, + "regoper[]": ArrayType, + "regoperator[]": ArrayType, + "regclass[]": ArrayType, + "regcollation[]": ArrayType, + "regtype[]": ArrayType, + "regrole[]": ArrayType, + "regnamespace[]": ArrayType, + "uuid[]": ArrayType, + "pg_lsn[]": ArrayType, + "tsvector[]": ArrayType, + "gtsvector[]": ArrayType, + "tsquery[]": ArrayType, + "regconfig[]": ArrayType, + "regdictionary[]": ArrayType, + "jsonb[]": ArrayType, + "jsonpath[]": ArrayType, + "txid_snapshot[]": ArrayType, + "pg_snapshot[]": ArrayType, + "int4range[]": ArrayType, + "numrange[]": ArrayType, + "tsrange[]": ArrayType, + "tstzrange[]": ArrayType, + "daterange[]": ArrayType, + "int8range[]": ArrayType, + "cstring[]": ArrayType, +} + +# Postgres types with modifiers (identifiable by non-empty typmodin and typmodout columns) +POSTGRES_MODIFIED_TYPES = { + "character varying", + "character varying[]", + "bit varying", + "bit varying[]", + "time with time zone", + "time with time zone[]", + "time without time zone", + "time without time zone[]", + "timestamp with time zone", + "timestamp with time zone[]", + "timestamp without time zone", + "timestamp without time zone[]", + "numeric", + "numeric[]", + "interval", + "interval[]", + "character", + "character[]", + "bit", + "bit[]", +} + + +def resolve_postgres_modified_type(type_string: str) -> Any: + + if type_string.endswith("[]"): + return ArrayType + + for modified_type_base in POSTGRES_MODIFIED_TYPES: + if re.match(rf"{re.escape(modified_type_base)}\([0-9,]+\)", type_string): + return POSTGRES_TYPES_MAP[modified_type_base] + + return None + + +# see https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html +SNOWFLAKE_TYPES_MAP: Dict[str, Any] = { + "NUMBER": NumberType, + "DECIMAL": NumberType, + "NUMERIC": NumberType, + "INT": NumberType, + "INTEGER": NumberType, + "BIGINT": NumberType, + "SMALLINT": NumberType, + "FLOAT": NumberType, + "FLOAT4": NumberType, + "FLOAT8": NumberType, + "DOUBLE": NumberType, + "DOUBLE PRECISION": NumberType, + "REAL": NumberType, + "VARCHAR": StringType, + "CHAR": StringType, + "CHARACTER": StringType, + "STRING": StringType, + "TEXT": StringType, + "BINARY": BytesType, + "VARBINARY": BytesType, + "BOOLEAN": BooleanType, + "DATE": DateType, + "DATETIME": DateType, + "TIME": TimeType, + "TIMESTAMP": TimeType, + "TIMESTAMP_LTZ": TimeType, + "TIMESTAMP_NTZ": TimeType, + "TIMESTAMP_TZ": TimeType, + "VARIANT": RecordType, + "OBJECT": RecordType, + "ARRAY": ArrayType, + "GEOGRAPHY": None, +} diff --git a/metadata-ingestion/tests/integration/dbt/dbt_catalog.json b/metadata-ingestion/tests/integration/dbt/dbt_catalog.json index 83536af910dc52..afc5e21b70238e 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_catalog.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_catalog.json @@ -1,704 +1,728 @@ { - "metadata": { - "dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", - "dbt_version": "0.19.0", - "generated_at": "2021-03-07T18:38:59.924004Z", - "invocation_id": "8f050331-a7c7-419f-ac33-829eb8a7611a", - "env": {} + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "dbt_version": "0.19.0", + "generated_at": "2021-03-07T18:38:59.924004Z", + "invocation_id": "8f050331-a7c7-419f-ac33-829eb8a7611a", + "env": {} + }, + "nodes": { + "model.sample_dbt.monthly_billing_with_cust": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "pagila_dev", + "name": "monthly_billing_with_cust", + "comment": null, + "owner": "postgres" + }, + "columns": { + "billing_month": { + "type": "timestamp with time zone", + "comment": null, + "index": 1, + "name": "billing_month" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "amount": { + "type": "numeric", + "comment": null, + "index": 3, + "name": "amount" + }, + "email": { + "type": "text", + "comment": null, + "index": 4, + "name": "email" + }, + "emails": { + "type": "text[]", + "comment": null, + "index": 5, + "name": "emails" + }, + "cost": { + "type": "money", + "comment": null, + "index": 6, + "name": "cost" + }, + "native": { + "type": "some native type", + "comment": null, + "index": 7, + "name": "native" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "model.sample_dbt.monthly_billing_with_cust" + }, + "model.sample_dbt.payments_base": { + "metadata": { + "type": "VIEW", + "database": "pagila", + "schema": "pagila_dev", + "name": "payments_base", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "model.sample_dbt.payments_base" + }, + "model.sample_dbt.payments_by_customer_by_month": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "pagila_dev", + "name": "payments_by_customer_by_month", + "comment": null, + "owner": "postgres" + }, + "columns": { + "billing_month": { + "type": "timestamp with time zone", + "comment": null, + "index": 1, + "name": "billing_month" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "amount": { + "type": "numeric", + "comment": null, + "index": 3, + "name": "amount" + }, + "name": { + "type": "character varying(129)", + "comment": null, + "index": 4, + "name": "name" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "model.sample_dbt.payments_by_customer_by_month" + } + }, + "sources": { + "source.sample_dbt.pagila.address": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "address", + "comment": null, + "owner": "postgres" + }, + "columns": { + "address_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "address_id" + }, + "address": { + "type": "text", + "comment": null, + "index": 2, + "name": "address" + }, + "address2": { + "type": "text", + "comment": null, + "index": 3, + "name": "address2" + }, + "district": { + "type": "text", + "comment": null, + "index": 4, + "name": "district" + }, + "city_id": { + "type": "integer", + "comment": null, + "index": 5, + "name": "city_id" + }, + "postal_code": { + "type": "text", + "comment": null, + "index": 6, + "name": "postal_code" + }, + "phone": { + "type": "text", + "comment": null, + "index": 7, + "name": "phone" + }, + "last_update": { + "type": "timestamp with time zone", + "comment": null, + "index": 8, + "name": "last_update" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.address" + }, + "source.sample_dbt.pagila.city": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "city", + "comment": null, + "owner": "postgres" + }, + "columns": { + "city_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "city_id" + }, + "city": { + "type": "text", + "comment": null, + "index": 2, + "name": "city" + }, + "country_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "country_id" + }, + "last_update": { + "type": "timestamp with time zone", + "comment": null, + "index": 4, + "name": "last_update" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.city" + }, + "source.sample_dbt.pagila.customer": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "customer", + "comment": null, + "owner": "postgres" + }, + "columns": { + "customer_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "customer_id" + }, + "store_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "store_id" + }, + "first_name": { + "type": "text", + "comment": null, + "index": 3, + "name": "first_name" + }, + "last_name": { + "type": "text", + "comment": null, + "index": 4, + "name": "last_name" + }, + "email": { + "type": "text", + "comment": null, + "index": 5, + "name": "email" + }, + "address_id": { + "type": "integer", + "comment": null, + "index": 6, + "name": "address_id" + }, + "activebool": { + "type": "boolean", + "comment": null, + "index": 7, + "name": "activebool" + }, + "create_date": { + "type": "date", + "comment": null, + "index": 8, + "name": "create_date" + }, + "last_update": { + "type": "timestamp with time zone", + "comment": null, + "index": 9, + "name": "last_update" + }, + "active": { + "type": "integer", + "comment": null, + "index": 10, + "name": "active" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.customer" + }, + "source.sample_dbt.pagila.payment_p2020_01": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "payment_p2020_01", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_01" + }, + "source.sample_dbt.pagila.payment_p2020_02": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "payment_p2020_02", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_02" + }, + "source.sample_dbt.pagila.payment_p2020_03": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "payment_p2020_03", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_03" }, - "nodes": { - "model.sample_dbt.monthly_billing_with_cust": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "pagila_dev", - "name": "monthly_billing_with_cust", - "comment": null, - "owner": "postgres" - }, - "columns": { - "billing_month": { - "type": "timestamp with time zone", - "comment": null, - "index": 1, - "name": "billing_month" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "amount": { - "type": "numeric", - "comment": null, - "index": 3, - "name": "amount" - }, - "email": { - "type": "text", - "comment": null, - "index": 4, - "name": "email" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "model.sample_dbt.monthly_billing_with_cust" - }, - "model.sample_dbt.payments_base": { - "metadata": { - "type": "VIEW", - "database": "pagila", - "schema": "pagila_dev", - "name": "payments_base", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "model.sample_dbt.payments_base" - }, - "model.sample_dbt.payments_by_customer_by_month": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "pagila_dev", - "name": "payments_by_customer_by_month", - "comment": null, - "owner": "postgres" - }, - "columns": { - "billing_month": { - "type": "timestamp with time zone", - "comment": null, - "index": 1, - "name": "billing_month" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "amount": { - "type": "numeric", - "comment": null, - "index": 3, - "name": "amount" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "model.sample_dbt.payments_by_customer_by_month" + "source.sample_dbt.pagila.payment_p2020_04": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "payment_p2020_04", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_04" }, - "sources": { - "source.sample_dbt.pagila.address": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "address", - "comment": null, - "owner": "postgres" - }, - "columns": { - "address_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "address_id" - }, - "address": { - "type": "text", - "comment": null, - "index": 2, - "name": "address" - }, - "address2": { - "type": "text", - "comment": null, - "index": 3, - "name": "address2" - }, - "district": { - "type": "text", - "comment": null, - "index": 4, - "name": "district" - }, - "city_id": { - "type": "integer", - "comment": null, - "index": 5, - "name": "city_id" - }, - "postal_code": { - "type": "text", - "comment": null, - "index": 6, - "name": "postal_code" - }, - "phone": { - "type": "text", - "comment": null, - "index": 7, - "name": "phone" - }, - "last_update": { - "type": "timestamp with time zone", - "comment": null, - "index": 8, - "name": "last_update" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.address" - }, - "source.sample_dbt.pagila.city": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "city", - "comment": null, - "owner": "postgres" - }, - "columns": { - "city_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "city_id" - }, - "city": { - "type": "text", - "comment": null, - "index": 2, - "name": "city" - }, - "country_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "country_id" - }, - "last_update": { - "type": "timestamp with time zone", - "comment": null, - "index": 4, - "name": "last_update" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.city" - }, - "source.sample_dbt.pagila.customer": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "customer", - "comment": null, - "owner": "postgres" - }, - "columns": { - "customer_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "customer_id" - }, - "store_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "store_id" - }, - "first_name": { - "type": "text", - "comment": null, - "index": 3, - "name": "first_name" - }, - "last_name": { - "type": "text", - "comment": null, - "index": 4, - "name": "last_name" - }, - "email": { - "type": "text", - "comment": null, - "index": 5, - "name": "email" - }, - "address_id": { - "type": "integer", - "comment": null, - "index": 6, - "name": "address_id" - }, - "activebool": { - "type": "boolean", - "comment": null, - "index": 7, - "name": "activebool" - }, - "create_date": { - "type": "date", - "comment": null, - "index": 8, - "name": "create_date" - }, - "last_update": { - "type": "timestamp with time zone", - "comment": null, - "index": 9, - "name": "last_update" - }, - "active": { - "type": "integer", - "comment": null, - "index": 10, - "name": "active" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.customer" - }, - "source.sample_dbt.pagila.payment_p2020_01": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "payment_p2020_01", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.payment_p2020_01" - }, - "source.sample_dbt.pagila.payment_p2020_02": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "payment_p2020_02", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.payment_p2020_02" - }, - "source.sample_dbt.pagila.payment_p2020_03": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "payment_p2020_03", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.payment_p2020_03" - }, - "source.sample_dbt.pagila.payment_p2020_04": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "payment_p2020_04", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.payment_p2020_04" - }, - "source.sample_dbt.pagila.payment_p2020_05": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "payment_p2020_05", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.payment_p2020_05" - }, - "source.sample_dbt.pagila.payment_p2020_06": { - "metadata": { - "type": "BASE TABLE", - "database": "pagila", - "schema": "public", - "name": "payment_p2020_06", - "comment": null, - "owner": "postgres" - }, - "columns": { - "payment_id": { - "type": "integer", - "comment": null, - "index": 1, - "name": "payment_id" - }, - "customer_id": { - "type": "integer", - "comment": null, - "index": 2, - "name": "customer_id" - }, - "staff_id": { - "type": "integer", - "comment": null, - "index": 3, - "name": "staff_id" - }, - "rental_id": { - "type": "integer", - "comment": null, - "index": 4, - "name": "rental_id" - }, - "amount": { - "type": "numeric(5,2)", - "comment": null, - "index": 5, - "name": "amount" - }, - "payment_date": { - "type": "timestamp with time zone", - "comment": null, - "index": 6, - "name": "payment_date" - } - }, - "stats": { - "has_stats": { - "id": "has_stats", - "label": "Has Stats?", - "value": false, - "description": "Indicates whether there are statistics for this table", - "include": false - } - }, - "unique_id": "source.sample_dbt.pagila.payment_p2020_06" + "source.sample_dbt.pagila.payment_p2020_05": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "payment_p2020_05", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_05" }, - "errors": null -} \ No newline at end of file + "source.sample_dbt.pagila.payment_p2020_06": { + "metadata": { + "type": "BASE TABLE", + "database": "pagila", + "schema": "public", + "name": "payment_p2020_06", + "comment": null, + "owner": "postgres" + }, + "columns": { + "payment_id": { + "type": "integer", + "comment": null, + "index": 1, + "name": "payment_id" + }, + "customer_id": { + "type": "integer", + "comment": null, + "index": 2, + "name": "customer_id" + }, + "staff_id": { + "type": "integer", + "comment": null, + "index": 3, + "name": "staff_id" + }, + "rental_id": { + "type": "integer", + "comment": null, + "index": 4, + "name": "rental_id" + }, + "amount": { + "type": "numeric(5,2)", + "comment": null, + "index": 5, + "name": "amount" + }, + "payment_date": { + "type": "timestamp with time zone", + "comment": null, + "index": 6, + "name": "payment_date" + } + }, + "stats": { + "has_stats": { + "id": "has_stats", + "label": "Has Stats?", + "value": false, + "description": "Indicates whether there are statistics for this table", + "include": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_06" + } + }, + "errors": null +} diff --git a/metadata-ingestion/tests/integration/dbt/dbt_manifest.json b/metadata-ingestion/tests/integration/dbt/dbt_manifest.json index cdf28a49ed7dcf..c1b5ad669082eb 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_manifest.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_manifest.json @@ -1,3740 +1,3630 @@ { - "metadata": { - "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v1.json", - "dbt_version": "0.19.0", - "generated_at": "2021-04-02T17:14:20.511467Z", - "invocation_id": "fc43f644-d50b-41bd-8252-2d1d45cd891e", - "env": {}, - "project_id": "d7ed27e3f02ff95ca753dd76aed208ee", - "user_id": "3e781a9c-6838-4db1-95b7-54625ba7a601", - "send_anonymous_usage_stats": true, - "adapter_type": "postgres" - }, - "nodes": { - "model.sample_dbt.payments_base": { - "raw_sql": "{{ config(\n materialized=\"view\",\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", - "compiled": true, - "database": "pagila", - "schema": "pagila_dev", - "fqn": [ - "sample_dbt", - "base", - "payments_base" - ], - "unique_id": "model.sample_dbt.payments_base", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "base/payments_base.sql", - "original_file_path": "models/base/payments_base.sql", - "name": "payments_base", - "resource_type": "model", - "alias": "payments_base", - "checksum": { - "name": "sha256", - "checksum": "1ad81ef75603c79e0fe3672cf4fdda70d39f39bbf52aa7f1085ed808afdaeb3d" - }, - "config": { - "enabled": true, - "materialized": "view", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "vars": {}, - "quoting": {}, - "column_types": {}, - "alias": null, - "schema": null, - "database": null, - "tags": [], - "full_refresh": null - }, - "tags": [], - "refs": [], - "sources": [ - [ - "pagila", - "payment_p2020_01" - ], - [ - "pagila", - "payment_p2020_02" - ], - [ - "pagila", - "payment_p2020_03" - ], - [ - "pagila", - "payment_p2020_04" - ], - [ - "pagila", - "payment_p2020_05" - ], - [ - "pagila", - "payment_p2020_06" - ] - ], - "depends_on": { - "macros": [ - "macro.dbt.run_hooks", - "macro.dbt.statement", - "macro.dbt.persist_docs", - "macro.dbt.drop_relation_if_exists" - ], - "nodes": [ - "source.sample_dbt.pagila.payment_p2020_01", - "source.sample_dbt.pagila.payment_p2020_02", - "source.sample_dbt.pagila.payment_p2020_03", - "source.sample_dbt.pagila.payment_p2020_04", - "source.sample_dbt.pagila.payment_p2020_05", - "source.sample_dbt.pagila.payment_p2020_06" - ] - }, - "description": "", - "columns": {}, - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "build_path": "target/run/sample_dbt/models/base/payments_base.sql", - "deferred": false, - "unrendered_config": { - "materialized": "view" - }, - "compiled_sql": "\n\nwith payments as (\n\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_06\"\n)\n\nselect *\nfrom payments", - "extra_ctes_injected": true, - "extra_ctes": [], - "relation_name": "\"pagila\".\"pagila_dev\".\"payments_base\"" - }, - "model.sample_dbt.payments_by_customer_by_month": { - "raw_sql": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n billing_month,\n customer_id", - "compiled": true, - "database": "pagila", - "schema": "pagila_dev", - "fqn": [ - "sample_dbt", - "transform", - "payments_by_customer_by_month" - ], - "unique_id": "model.sample_dbt.payments_by_customer_by_month", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "transform/payments_by_customer_by_month.sql", - "original_file_path": "models/transform/payments_by_customer_by_month.sql", - "name": "payments_by_customer_by_month", - "resource_type": "model", - "alias": "payments_by_customer_by_month", - "checksum": { - "name": "sha256", - "checksum": "691757217fdc0ad849160f0b24bff8fa09385b51f9f5605dda388a6d47d1ab28" - }, - "config": { - "enabled": true, - "materialized": "table", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "vars": {}, - "quoting": {}, - "column_types": {}, - "alias": null, - "schema": null, - "database": null, - "tags": [], - "full_refresh": null - }, - "tags": [], - "refs": [ - [ - "payments_base" - ] - ], - "sources": [], - "depends_on": { - "macros": [ - "macro.dbt.run_hooks", - "macro.dbt.statement", - "macro.dbt.persist_docs", - "macro.dbt.drop_relation_if_exists" - ], - "nodes": [ - "model.sample_dbt.payments_base" - ] - }, - "description": "", - "columns": {}, - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "build_path": "target/run/sample_dbt/models/transform/payments_by_customer_by_month.sql", - "deferred": false, - "unrendered_config": { - "materialized": "table" - }, - "compiled_sql": "\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n \"pagila\".\"pagila_dev\".\"payments_base\"\nGROUP BY\n billing_month,\n customer_id", - "extra_ctes_injected": true, - "extra_ctes": [], - "relation_name": "\"pagila\".\"pagila_dev\".\"payments_by_customer_by_month\"" - }, - "model.sample_dbt.customer_details": { - "raw_sql": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", - "compiled": true, - "database": "pagila", - "schema": "pagila_dev", - "fqn": [ - "sample_dbt", - "transform", - "customer_details" - ], - "unique_id": "model.sample_dbt.customer_details", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "transform/customer_details.sql", - "original_file_path": "models/transform/customer_details.sql", - "name": "customer_details", - "resource_type": "model", - "alias": "customer_details", - "checksum": { - "name": "sha256", - "checksum": "038ac5af10d168699b8e5dcef54752c22a31a705eed010cacbf180e9ca214ceb" - }, - "config": { - "enabled": true, - "materialized": "ephemeral", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "vars": {}, - "quoting": {}, - "column_types": {}, - "alias": null, - "schema": null, - "database": null, - "tags": [], - "full_refresh": null - }, - "tags": [], - "refs": [], - "sources": [ - [ - "pagila", - "customer" - ], - [ - "pagila", - "address" - ], - [ - "pagila", - "city" - ] - ], - "depends_on": { - "macros": [], - "nodes": [ - "source.sample_dbt.pagila.customer", - "source.sample_dbt.pagila.address", - "source.sample_dbt.pagila.city" - ] - }, - "description": "", - "columns": {}, - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "build_path": "target/compiled/sample_dbt/models/transform/customer_details.sql", - "deferred": false, - "unrendered_config": { - "materialized": "ephemeral" - }, - "compiled_sql": "\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id", - "extra_ctes_injected": true, - "extra_ctes": [], - "relation_name": null - }, - "model.sample_dbt.monthly_billing_with_cust": { - "raw_sql": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", - "compiled": true, - "database": "pagila", - "schema": "pagila_dev", - "fqn": [ - "sample_dbt", - "billing", - "monthly_billing_with_cust" - ], - "unique_id": "model.sample_dbt.monthly_billing_with_cust", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "billing/monthly_billing_with_cust.sql", - "original_file_path": "models/billing/monthly_billing_with_cust.sql", - "name": "monthly_billing_with_cust", - "resource_type": "model", - "alias": "monthly_billing_with_cust", - "checksum": { - "name": "sha256", - "checksum": "96468f0564c87d3dfb75f80f0bf1dac169f67137dd46eeefdb2a9939a03fad13" - }, - "config": { - "enabled": true, - "materialized": "table", - "persist_docs": {}, - "post-hook": [], - "pre-hook": [], - "vars": {}, - "quoting": {}, - "column_types": {}, - "alias": null, - "schema": null, - "database": null, - "tags": [], - "full_refresh": null - }, - "tags": [], - "refs": [ - [ - "payments_by_customer_by_month" - ], - [ - "customer_details" - ] - ], - "sources": [], - "depends_on": { - "macros": [ - "macro.dbt.run_hooks", - "macro.dbt.statement", - "macro.dbt.persist_docs", - "macro.dbt.drop_relation_if_exists" - ], - "nodes": [ - "model.sample_dbt.payments_by_customer_by_month", - "model.sample_dbt.customer_details" - ] - }, - "description": "", - "columns": {}, - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "build_path": "target/run/sample_dbt/models/billing/monthly_billing_with_cust.sql", - "deferred": false, - "unrendered_config": { - "materialized": "table" - }, - "compiled_sql": "\n\nwith __dbt__CTE__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)SELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n \"pagila\".\"pagila_dev\".\"payments_by_customer_by_month\" pbc\n left outer join __dbt__CTE__customer_details cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", - "extra_ctes_injected": true, - "extra_ctes": [ - { - "id": "model.sample_dbt.customer_details", - "sql": " __dbt__CTE__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)" - } - ], - "relation_name": "\"pagila\".\"pagila_dev\".\"monthly_billing_with_cust\"" - } + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v1.json", + "dbt_version": "0.19.0", + "generated_at": "2021-04-02T17:14:20.511467Z", + "invocation_id": "fc43f644-d50b-41bd-8252-2d1d45cd891e", + "env": {}, + "project_id": "d7ed27e3f02ff95ca753dd76aed208ee", + "user_id": "3e781a9c-6838-4db1-95b7-54625ba7a601", + "send_anonymous_usage_stats": true, + "adapter_type": "postgres" + }, + "nodes": { + "model.sample_dbt.payments_base": { + "raw_sql": "{{ config(\n materialized=\"view\",\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", + "compiled": true, + "database": "pagila", + "schema": "pagila_dev", + "fqn": ["sample_dbt", "base", "payments_base"], + "unique_id": "model.sample_dbt.payments_base", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "base/payments_base.sql", + "original_file_path": "models/base/payments_base.sql", + "name": "payments_base", + "resource_type": "model", + "alias": "payments_base", + "checksum": { + "name": "sha256", + "checksum": "1ad81ef75603c79e0fe3672cf4fdda70d39f39bbf52aa7f1085ed808afdaeb3d" + }, + "config": { + "enabled": true, + "materialized": "view", + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "vars": {}, + "quoting": {}, + "column_types": {}, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "full_refresh": null + }, + "tags": [], + "refs": [], + "sources": [ + ["pagila", "payment_p2020_01"], + ["pagila", "payment_p2020_02"], + ["pagila", "payment_p2020_03"], + ["pagila", "payment_p2020_04"], + ["pagila", "payment_p2020_05"], + ["pagila", "payment_p2020_06"] + ], + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.persist_docs", + "macro.dbt.drop_relation_if_exists" + ], + "nodes": [ + "source.sample_dbt.pagila.payment_p2020_01", + "source.sample_dbt.pagila.payment_p2020_02", + "source.sample_dbt.pagila.payment_p2020_03", + "source.sample_dbt.pagila.payment_p2020_04", + "source.sample_dbt.pagila.payment_p2020_05", + "source.sample_dbt.pagila.payment_p2020_06" + ] + }, + "description": "", + "columns": {}, + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "build_path": "target/run/sample_dbt/models/base/payments_base.sql", + "deferred": false, + "unrendered_config": { + "materialized": "view" + }, + "compiled_sql": "\n\nwith payments as (\n\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_06\"\n)\n\nselect *\nfrom payments", + "extra_ctes_injected": true, + "extra_ctes": [], + "relation_name": "\"pagila\".\"pagila_dev\".\"payments_base\"" }, - "sources": { - "source.sample_dbt.pagila.payment_p2020_01": { - "fqn": [ - "sample_dbt", - "pagila", - "payment_p2020_01" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.payment_p2020_01", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "payment_p2020_01", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "payment_p2020_01", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"payment_p2020_01\"" - }, - "source.sample_dbt.pagila.payment_p2020_02": { - "fqn": [ - "sample_dbt", - "pagila", - "payment_p2020_02" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.payment_p2020_02", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "payment_p2020_02", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "payment_p2020_02", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"payment_p2020_02\"" - }, - "source.sample_dbt.pagila.payment_p2020_03": { - "fqn": [ - "sample_dbt", - "pagila", - "payment_p2020_03" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.payment_p2020_03", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "payment_p2020_03", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "payment_p2020_03", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"payment_p2020_03\"" - }, - "source.sample_dbt.pagila.payment_p2020_04": { - "fqn": [ - "sample_dbt", - "pagila", - "payment_p2020_04" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.payment_p2020_04", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "payment_p2020_04", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "payment_p2020_04", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"payment_p2020_04\"" - }, - "source.sample_dbt.pagila.payment_p2020_05": { - "fqn": [ - "sample_dbt", - "pagila", - "payment_p2020_05" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.payment_p2020_05", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "payment_p2020_05", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "payment_p2020_05", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"payment_p2020_05\"" - }, - "source.sample_dbt.pagila.payment_p2020_06": { - "fqn": [ - "sample_dbt", - "pagila", - "payment_p2020_06" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.payment_p2020_06", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "payment_p2020_06", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "payment_p2020_06", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"payment_p2020_06\"" - }, - "source.sample_dbt.pagila.customer": { - "fqn": [ - "sample_dbt", - "pagila", - "customer" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.customer", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "customer", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "customer", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"customer\"" - }, - "source.sample_dbt.pagila.address": { - "fqn": [ - "sample_dbt", - "pagila", - "address" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.address", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "address", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "address", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"address\"" - }, - "source.sample_dbt.pagila.city": { - "fqn": [ - "sample_dbt", - "pagila", - "city" - ], - "database": "pagila", - "schema": "public", - "unique_id": "source.sample_dbt.pagila.city", - "package_name": "sample_dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt", - "path": "models/base.yml", - "original_file_path": "models/base.yml", - "name": "city", - "source_name": "pagila", - "source_description": "base models", - "loader": "", - "identifier": "city", - "resource_type": "source", - "quoting": { - "database": null, - "schema": null, - "identifier": null, - "column": null - }, - "loaded_at_field": null, - "freshness": { - "warn_after": null, - "error_after": null, - "filter": null - }, - "external": null, - "description": "", - "columns": {}, - "meta": {}, - "source_meta": {}, - "tags": [], - "config": { - "enabled": true - }, - "patch_path": null, - "unrendered_config": {}, - "relation_name": "\"pagila\".\"public\".\"city\"" - } + "model.sample_dbt.payments_by_customer_by_month": { + "raw_sql": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n billing_month,\n customer_id", + "compiled": true, + "database": "pagila", + "schema": "pagila_dev", + "fqn": ["sample_dbt", "transform", "payments_by_customer_by_month"], + "unique_id": "model.sample_dbt.payments_by_customer_by_month", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "transform/payments_by_customer_by_month.sql", + "original_file_path": "models/transform/payments_by_customer_by_month.sql", + "name": "payments_by_customer_by_month", + "resource_type": "model", + "alias": "payments_by_customer_by_month", + "checksum": { + "name": "sha256", + "checksum": "691757217fdc0ad849160f0b24bff8fa09385b51f9f5605dda388a6d47d1ab28" + }, + "config": { + "enabled": true, + "materialized": "table", + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "vars": {}, + "quoting": {}, + "column_types": {}, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "full_refresh": null + }, + "tags": [], + "refs": [["payments_base"]], + "sources": [], + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.persist_docs", + "macro.dbt.drop_relation_if_exists" + ], + "nodes": ["model.sample_dbt.payments_base"] + }, + "description": "", + "columns": {}, + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "build_path": "target/run/sample_dbt/models/transform/payments_by_customer_by_month.sql", + "deferred": false, + "unrendered_config": { + "materialized": "table" + }, + "compiled_sql": "\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n \"pagila\".\"pagila_dev\".\"payments_base\"\nGROUP BY\n billing_month,\n customer_id", + "extra_ctes_injected": true, + "extra_ctes": [], + "relation_name": "\"pagila\".\"pagila_dev\".\"payments_by_customer_by_month\"" }, - "macros": { - "macro.dbt_postgres.postgres_get_relations": { - "unique_id": "macro.dbt_postgres.postgres_get_relations", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/relations.sql", - "original_file_path": "macros/relations.sql", - "name": "postgres_get_relations", - "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt.statement" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__get_catalog": { - "unique_id": "macro.dbt_postgres.postgres__get_catalog", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/catalog.sql", - "original_file_path": "macros/catalog.sql", - "name": "postgres__get_catalog", - "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence = 'p' -- [p]ermanent table. Other values are [u]nlogged table, [t]emporary table\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__create_table_as": { - "unique_id": "macro.dbt_postgres.postgres__create_table_as", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__create_table_as", - "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__create_schema": { - "unique_id": "macro.dbt_postgres.postgres__create_schema", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__create_schema", - "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__drop_schema": { - "unique_id": "macro.dbt_postgres.postgres__drop_schema", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__drop_schema", - "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__get_columns_in_relation": { - "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__get_columns_in_relation", - "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__list_relations_without_caching": { - "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__list_relations_without_caching", - "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__information_schema_name": { - "unique_id": "macro.dbt_postgres.postgres__information_schema_name", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__information_schema_name", - "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__list_schemas": { - "unique_id": "macro.dbt_postgres.postgres__list_schemas", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__list_schemas", - "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__check_schema_exists": { - "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__check_schema_exists", - "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__current_timestamp": { - "unique_id": "macro.dbt_postgres.postgres__current_timestamp", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__current_timestamp", - "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__snapshot_string_as_time": { - "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__snapshot_string_as_time", - "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__snapshot_get_time": { - "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__snapshot_get_time", - "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__make_temp_relation": { - "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__make_temp_relation", - "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres_escape_comment": { - "unique_id": "macro.dbt_postgres.postgres_escape_comment", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres_escape_comment", - "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__alter_relation_comment": { - "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__alter_relation_comment", - "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__alter_column_comment": { - "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/adapters.sql", - "original_file_path": "macros/adapters.sql", - "name": "postgres__alter_column_comment", - "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt_postgres.postgres__snapshot_merge_sql": { - "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", - "package_name": "dbt_postgres", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", - "path": "macros/materializations/snapshot_merge.sql", - "original_file_path": "macros/materializations/snapshot_merge.sql", - "name": "postgres__snapshot_merge_sql", - "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.statement": { - "unique_id": "macro.dbt.statement", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/core.sql", - "original_file_path": "macros/core.sql", - "name": "statement", - "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.noop_statement": { - "unique_id": "macro.dbt.noop_statement", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/core.sql", - "original_file_path": "macros/core.sql", - "name": "noop_statement", - "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.run_hooks": { - "unique_id": "macro.dbt.run_hooks", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "run_hooks", - "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.column_list": { - "unique_id": "macro.dbt.column_list", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "column_list", - "macro_sql": "{% macro column_list(columns) %}\n {%- for col in columns %}\n {{ col.name }} {% if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.column_list_for_create_table": { - "unique_id": "macro.dbt.column_list_for_create_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "column_list_for_create_table", - "macro_sql": "{% macro column_list_for_create_table(columns) %}\n {%- for col in columns %}\n {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.make_hook_config": { - "unique_id": "macro.dbt.make_hook_config", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "make_hook_config", - "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.before_begin": { - "unique_id": "macro.dbt.before_begin", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "before_begin", - "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.in_transaction": { - "unique_id": "macro.dbt.in_transaction", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "in_transaction", - "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.after_commit": { - "unique_id": "macro.dbt.after_commit", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "after_commit", - "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.drop_relation_if_exists": { - "unique_id": "macro.dbt.drop_relation_if_exists", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "drop_relation_if_exists", - "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.load_relation": { - "unique_id": "macro.dbt.load_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "load_relation", - "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.should_full_refresh": { - "unique_id": "macro.dbt.should_full_refresh", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/helpers.sql", - "original_file_path": "macros/materializations/helpers.sql", - "name": "should_full_refresh", - "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_merge_sql": { - "unique_id": "macro.dbt.get_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "get_merge_sql", - "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_delete_insert_merge_sql": { - "unique_id": "macro.dbt.get_delete_insert_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "get_delete_insert_merge_sql", - "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_insert_overwrite_merge_sql": { - "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "get_insert_overwrite_merge_sql", - "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__get_merge_sql": { - "unique_id": "macro.dbt.default__get_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "default__get_merge_sql", - "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column in dest_columns -%}\n {{ adapter.quote(column.name) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(column.name) }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_quoted_csv": { - "unique_id": "macro.dbt.get_quoted_csv", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "get_quoted_csv", - "macro_sql": "{% macro get_quoted_csv(column_names) %}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.common_get_delete_insert_merge_sql": { - "unique_id": "macro.dbt.common_get_delete_insert_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "common_get_delete_insert_merge_sql", - "macro_sql": "{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key is not none %}\n delete from {{ target }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n );\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__get_delete_insert_merge_sql": { - "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "default__get_delete_insert_merge_sql", - "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__get_insert_overwrite_merge_sql": { - "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/common/merge.sql", - "original_file_path": "macros/materializations/common/merge.sql", - "name": "default__get_insert_overwrite_merge_sql", - "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.materialization_table_default": { - "unique_id": "macro.dbt.materialization_table_default", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/table/table.sql", - "original_file_path": "macros/materializations/table/table.sql", - "name": "materialization_table_default", - "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_table_as(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.create_csv_table": { - "unique_id": "macro.dbt.create_csv_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "create_csv_table", - "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table')(model, agate_table) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.reset_csv_table": { - "unique_id": "macro.dbt.reset_csv_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "reset_csv_table", - "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.load_csv_rows": { - "unique_id": "macro.dbt.load_csv_rows", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "load_csv_rows", - "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__create_csv_table": { - "unique_id": "macro.dbt.default__create_csv_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "default__create_csv_table", - "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__reset_csv_table": { - "unique_id": "macro.dbt.default__reset_csv_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "default__reset_csv_table", - "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_seed_column_quoted_csv": { - "unique_id": "macro.dbt.get_seed_column_quoted_csv", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "get_seed_column_quoted_csv", - "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.basic_load_csv_rows": { - "unique_id": "macro.dbt.basic_load_csv_rows", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "basic_load_csv_rows", - "macro_sql": "{% macro basic_load_csv_rows(model, batch_size, agate_table) %}\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n %s\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__load_csv_rows": { - "unique_id": "macro.dbt.default__load_csv_rows", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "default__load_csv_rows", - "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.materialization_seed_default": { - "unique_id": "macro.dbt.materialization_seed_default", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/seed/seed.sql", - "original_file_path": "macros/materializations/seed/seed.sql", - "name": "materialization_seed_default", - "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.materialization_view_default": { - "unique_id": "macro.dbt.materialization_view_default", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/view/view.sql", - "original_file_path": "macros/materializations/view/view.sql", - "name": "materialization_view_default", - "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.handle_existing_table": { - "unique_id": "macro.dbt.handle_existing_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/view/create_or_replace_view.sql", - "original_file_path": "macros/materializations/view/create_or_replace_view.sql", - "name": "handle_existing_table", - "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch(\"handle_existing_table\", packages=['dbt'])(full_refresh, old_relation) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__handle_existing_table": { - "unique_id": "macro.dbt.default__handle_existing_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/view/create_or_replace_view.sql", - "original_file_path": "macros/materializations/view/create_or_replace_view.sql", - "name": "default__handle_existing_table", - "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.create_or_replace_view": { - "unique_id": "macro.dbt.create_or_replace_view", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/view/create_or_replace_view.sql", - "original_file_path": "macros/materializations/view/create_or_replace_view.sql", - "name": "create_or_replace_view", - "macro_sql": "{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {% if run_outside_transaction_hooks %}\n -- no transactions on BigQuery\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n {% endif %}\n\n -- `BEGIN` happens here on Snowflake\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if run_outside_transaction_hooks %}\n -- No transactions on BigQuery\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.materialization_incremental_default": { - "unique_id": "macro.dbt.materialization_incremental_default", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/incremental/incremental.sql", - "original_file_path": "macros/materializations/incremental/incremental.sql", - "name": "materialization_incremental_default", - "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(this) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% elif existing_relation.is_view or should_full_refresh() %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set backup_identifier = existing_relation.identifier ~ \"__dbt_backup\" %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n {% do adapter.drop_relation(backup_relation) %}\n\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.incremental_upsert": { - "unique_id": "macro.dbt.incremental_upsert", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/incremental/helpers.sql", - "original_file_path": "macros/materializations/incremental/helpers.sql", - "name": "incremental_upsert", - "macro_sql": "{% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name=\"main\") %}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n\n {%- if unique_key is not none -%}\n delete\n from {{ target_relation }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ tmp_relation }}\n );\n {%- endif %}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ tmp_relation }}\n );\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.create_columns": { - "unique_id": "macro.dbt.create_columns", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "create_columns", - "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns')(relation, columns) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__create_columns": { - "unique_id": "macro.dbt.default__create_columns", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "default__create_columns", - "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.post_snapshot": { - "unique_id": "macro.dbt.post_snapshot", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "post_snapshot", - "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot')(staging_relation) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__post_snapshot": { - "unique_id": "macro.dbt.default__post_snapshot", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "default__post_snapshot", - "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_staging_table": { - "unique_id": "macro.dbt.snapshot_staging_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "snapshot_staging_table", - "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select \n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n \n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n \n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.build_snapshot_table": { - "unique_id": "macro.dbt.build_snapshot_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "build_snapshot_table", - "macro_sql": "{% macro build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_or_create_relation": { - "unique_id": "macro.dbt.get_or_create_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "get_or_create_relation", - "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.build_snapshot_staging_table": { - "unique_id": "macro.dbt.build_snapshot_staging_table", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "build_snapshot_staging_table", - "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.materialization_snapshot_default": { - "unique_id": "macro.dbt.materialization_snapshot_default", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot.sql", - "original_file_path": "macros/materializations/snapshot/snapshot.sql", - "name": "materialization_snapshot_default", - "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.database, model.schema) %}\n {% endif %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.strategy_dispatch": { - "unique_id": "macro.dbt.strategy_dispatch", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "strategy_dispatch", - "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_hash_arguments": { - "unique_id": "macro.dbt.snapshot_hash_arguments", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "snapshot_hash_arguments", - "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments')(args) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__snapshot_hash_arguments": { - "unique_id": "macro.dbt.default__snapshot_hash_arguments", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "default__snapshot_hash_arguments", - "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_get_time": { - "unique_id": "macro.dbt.snapshot_get_time", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "snapshot_get_time", - "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time')() }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__snapshot_get_time": { - "unique_id": "macro.dbt.default__snapshot_get_time", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "default__snapshot_get_time", - "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_timestamp_strategy": { - "unique_id": "macro.dbt.snapshot_timestamp_strategy", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "snapshot_timestamp_strategy", - "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/fishtown-analytics/dbt/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_string_as_time": { - "unique_id": "macro.dbt.snapshot_string_as_time", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "snapshot_string_as_time", - "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time')(timestamp) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__snapshot_string_as_time": { - "unique_id": "macro.dbt.default__snapshot_string_as_time", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "default__snapshot_string_as_time", - "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_check_all_get_existing_columns": { - "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "snapshot_check_all_get_existing_columns", - "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_check_strategy": { - "unique_id": "macro.dbt.snapshot_check_strategy", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/strategies.sql", - "original_file_path": "macros/materializations/snapshot/strategies.sql", - "name": "snapshot_check_strategy", - "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n \n {% set select_current_time -%}\n select {{ snapshot_get_time() }} as snapshot_start\n {%- endset %}\n\n {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #}\n {%- set now = run_query(select_current_time)[0][0] -%}\n {% if now is none or now is undefined -%}\n {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}\n {%- endif %}\n {% set updated_at = snapshot_string_as_time(now) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n TRUE\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.snapshot_merge_sql": { - "unique_id": "macro.dbt.snapshot_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot_merge.sql", - "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", - "name": "snapshot_merge_sql", - "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql')(target, source, insert_cols) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__snapshot_merge_sql": { - "unique_id": "macro.dbt.default__snapshot_merge_sql", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/materializations/snapshot/snapshot_merge.sql", - "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", - "name": "default__snapshot_merge_sql", - "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n ;\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_columns_in_query": { - "unique_id": "macro.dbt.get_columns_in_query", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "get_columns_in_query", - "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__get_columns_in_query": { - "unique_id": "macro.dbt.default__get_columns_in_query", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__get_columns_in_query", - "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.create_schema": { - "unique_id": "macro.dbt.create_schema", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "create_schema", - "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema')(relation) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__create_schema" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__create_schema": { - "unique_id": "macro.dbt.default__create_schema", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__create_schema", - "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.drop_schema": { - "unique_id": "macro.dbt.drop_schema", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "drop_schema", - "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema')(relation) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__drop_schema": { - "unique_id": "macro.dbt.default__drop_schema", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__drop_schema", - "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.create_table_as": { - "unique_id": "macro.dbt.create_table_as", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "create_table_as", - "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as')(temporary, relation, sql) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__create_table_as": { - "unique_id": "macro.dbt.default__create_table_as", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__create_table_as", - "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.create_view_as": { - "unique_id": "macro.dbt.create_view_as", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "create_view_as", - "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as')(relation, sql) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__create_view_as": { - "unique_id": "macro.dbt.default__create_view_as", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__create_view_as", - "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_catalog": { - "unique_id": "macro.dbt.get_catalog", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "get_catalog", - "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog')(information_schema, schemas)) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__get_catalog": { - "unique_id": "macro.dbt.default__get_catalog", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__get_catalog", - "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.get_columns_in_relation": { - "unique_id": "macro.dbt.get_columns_in_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "get_columns_in_relation", - "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation')(relation)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.sql_convert_columns_in_relation": { - "unique_id": "macro.dbt.sql_convert_columns_in_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "sql_convert_columns_in_relation", - "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__get_columns_in_relation": { - "unique_id": "macro.dbt.default__get_columns_in_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__get_columns_in_relation", - "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.alter_column_type": { - "unique_id": "macro.dbt.alter_column_type", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "alter_column_type", - "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type')(relation, column_name, new_column_type)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.alter_column_comment": { - "unique_id": "macro.dbt.alter_column_comment", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "alter_column_comment", - "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment')(relation, column_dict)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__alter_column_comment": { - "unique_id": "macro.dbt.default__alter_column_comment", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__alter_column_comment", - "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.alter_relation_comment": { - "unique_id": "macro.dbt.alter_relation_comment", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "alter_relation_comment", - "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment')(relation, relation_comment)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__alter_relation_comment": { - "unique_id": "macro.dbt.default__alter_relation_comment", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__alter_relation_comment", - "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.persist_docs": { - "unique_id": "macro.dbt.persist_docs", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "persist_docs", - "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__persist_docs": { - "unique_id": "macro.dbt.default__persist_docs", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__persist_docs", - "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__alter_column_type": { - "unique_id": "macro.dbt.default__alter_column_type", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__alter_column_type", - "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.drop_relation": { - "unique_id": "macro.dbt.drop_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "drop_relation", - "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation')(relation)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt.default__drop_relation" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__drop_relation": { - "unique_id": "macro.dbt.default__drop_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__drop_relation", - "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.truncate_relation": { - "unique_id": "macro.dbt.truncate_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "truncate_relation", - "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation')(relation)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__truncate_relation": { - "unique_id": "macro.dbt.default__truncate_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__truncate_relation", - "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.rename_relation": { - "unique_id": "macro.dbt.rename_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "rename_relation", - "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation')(from_relation, to_relation)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt.default__rename_relation" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__rename_relation": { - "unique_id": "macro.dbt.default__rename_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__rename_relation", - "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.information_schema_name": { - "unique_id": "macro.dbt.information_schema_name", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "information_schema_name", - "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name')(database)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__information_schema_name": { - "unique_id": "macro.dbt.default__information_schema_name", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__information_schema_name", - "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.list_schemas": { - "unique_id": "macro.dbt.list_schemas", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "list_schemas", - "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas')(database)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__list_schemas" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__list_schemas": { - "unique_id": "macro.dbt.default__list_schemas", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__list_schemas", - "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.check_schema_exists": { - "unique_id": "macro.dbt.check_schema_exists", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "check_schema_exists", - "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists')(information_schema, schema)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__check_schema_exists": { - "unique_id": "macro.dbt.default__check_schema_exists", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__check_schema_exists", - "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.list_relations_without_caching": { - "unique_id": "macro.dbt.list_relations_without_caching", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "list_relations_without_caching", - "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching')(schema_relation)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt_postgres.postgres__list_relations_without_caching" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__list_relations_without_caching": { - "unique_id": "macro.dbt.default__list_relations_without_caching", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__list_relations_without_caching", - "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.current_timestamp": { - "unique_id": "macro.dbt.current_timestamp", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "current_timestamp", - "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp')() }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__current_timestamp": { - "unique_id": "macro.dbt.default__current_timestamp", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__current_timestamp", - "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.collect_freshness": { - "unique_id": "macro.dbt.collect_freshness", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "collect_freshness", - "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness')(source, loaded_at_field, filter))}}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__collect_freshness": { - "unique_id": "macro.dbt.default__collect_freshness", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__collect_freshness", - "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.make_temp_relation": { - "unique_id": "macro.dbt.make_temp_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "make_temp_relation", - "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation')(base_relation, suffix))}}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__make_temp_relation": { - "unique_id": "macro.dbt.default__make_temp_relation", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "default__make_temp_relation", - "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.set_sql_header": { - "unique_id": "macro.dbt.set_sql_header", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/adapters/common.sql", - "original_file_path": "macros/adapters/common.sql", - "name": "set_sql_header", - "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__test_relationships": { - "unique_id": "macro.dbt.default__test_relationships", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/relationships.sql", - "original_file_path": "macros/schema_tests/relationships.sql", - "name": "default__test_relationships", - "macro_sql": "{% macro default__test_relationships(model, to, field) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}\n\n\nselect count(*) as validation_errors\nfrom (\n select {{ column_name }} as id from {{ model }}\n) as child\nleft join (\n select {{ field }} as id from {{ to }}\n) as parent on parent.id = child.id\nwhere child.id is not null\n and parent.id is null\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.test_relationships": { - "unique_id": "macro.dbt.test_relationships", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/relationships.sql", - "original_file_path": "macros/schema_tests/relationships.sql", - "name": "test_relationships", - "macro_sql": "{% macro test_relationships(model, to, field) %}\n {% set macro = adapter.dispatch('test_relationships') %}\n {{ macro(model, to, field, **kwargs) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__test_not_null": { - "unique_id": "macro.dbt.default__test_not_null", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/not_null.sql", - "original_file_path": "macros/schema_tests/not_null.sql", - "name": "default__test_not_null", - "macro_sql": "{% macro default__test_not_null(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n\nselect count(*) as validation_errors\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.test_not_null": { - "unique_id": "macro.dbt.test_not_null", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/not_null.sql", - "original_file_path": "macros/schema_tests/not_null.sql", - "name": "test_not_null", - "macro_sql": "{% macro test_not_null(model) %}\n {% set macro = adapter.dispatch('test_not_null') %}\n {{ macro(model, **kwargs) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__test_unique": { - "unique_id": "macro.dbt.default__test_unique", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/unique.sql", - "original_file_path": "macros/schema_tests/unique.sql", - "name": "default__test_unique", - "macro_sql": "{% macro default__test_unique(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n\nselect count(*) as validation_errors\nfrom (\n\n select\n {{ column_name }}\n\n from {{ model }}\n where {{ column_name }} is not null\n group by {{ column_name }}\n having count(*) > 1\n\n) validation_errors\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.test_unique": { - "unique_id": "macro.dbt.test_unique", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/unique.sql", - "original_file_path": "macros/schema_tests/unique.sql", - "name": "test_unique", - "macro_sql": "{% macro test_unique(model) %}\n {% set macro = adapter.dispatch('test_unique') %}\n {{ macro(model, **kwargs) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__test_accepted_values": { - "unique_id": "macro.dbt.default__test_accepted_values", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/accepted_values.sql", - "original_file_path": "macros/schema_tests/accepted_values.sql", - "name": "default__test_accepted_values", - "macro_sql": "{% macro default__test_accepted_values(model, values) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}\n{% set quote_values = kwargs.get('quote', True) %}\n\nwith all_values as (\n\n select distinct\n {{ column_name }} as value_field\n\n from {{ model }}\n\n),\n\nvalidation_errors as (\n\n select\n value_field\n\n from all_values\n where value_field not in (\n {% for value in values -%}\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n )\n)\n\nselect count(*) as validation_errors\nfrom validation_errors\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.test_accepted_values": { - "unique_id": "macro.dbt.test_accepted_values", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/schema_tests/accepted_values.sql", - "original_file_path": "macros/schema_tests/accepted_values.sql", - "name": "test_accepted_values", - "macro_sql": "{% macro test_accepted_values(model, values) %}\n {% set macro = adapter.dispatch('test_accepted_values') %}\n {{ macro(model, values, **kwargs) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.is_incremental": { - "unique_id": "macro.dbt.is_incremental", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/is_incremental.sql", - "original_file_path": "macros/etc/is_incremental.sql", - "name": "is_incremental", - "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.run_query": { - "unique_id": "macro.dbt.run_query", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/query.sql", - "original_file_path": "macros/etc/query.sql", - "name": "run_query", - "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.generate_database_name": { - "unique_id": "macro.dbt.generate_database_name", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/get_custom_database.sql", - "original_file_path": "macros/etc/get_custom_database.sql", - "name": "generate_database_name", - "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name')(custom_database_name, node)) %}\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [ - "macro.dbt.default__generate_database_name" - ] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.default__generate_database_name": { - "unique_id": "macro.dbt.default__generate_database_name", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/get_custom_database.sql", - "original_file_path": "macros/etc/get_custom_database.sql", - "name": "default__generate_database_name", - "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.generate_alias_name": { - "unique_id": "macro.dbt.generate_alias_name", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/get_custom_alias.sql", - "original_file_path": "macros/etc/get_custom_alias.sql", - "name": "generate_alias_name", - "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.generate_schema_name": { - "unique_id": "macro.dbt.generate_schema_name", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/get_custom_schema.sql", - "original_file_path": "macros/etc/get_custom_schema.sql", - "name": "generate_schema_name", - "macro_sql": "{% macro generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.generate_schema_name_for_env": { - "unique_id": "macro.dbt.generate_schema_name_for_env", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/get_custom_schema.sql", - "original_file_path": "macros/etc/get_custom_schema.sql", - "name": "generate_schema_name_for_env", - "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.convert_datetime": { - "unique_id": "macro.dbt.convert_datetime", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/datetime.sql", - "original_file_path": "macros/etc/datetime.sql", - "name": "convert_datetime", - "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.dates_in_range": { - "unique_id": "macro.dbt.dates_in_range", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/datetime.sql", - "original_file_path": "macros/etc/datetime.sql", - "name": "dates_in_range", - "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.partition_range": { - "unique_id": "macro.dbt.partition_range", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/datetime.sql", - "original_file_path": "macros/etc/datetime.sql", - "name": "partition_range", - "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - }, - "macro.dbt.py_current_timestring": { - "unique_id": "macro.dbt.py_current_timestring", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "macros/etc/datetime.sql", - "original_file_path": "macros/etc/datetime.sql", - "name": "py_current_timestring", - "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", - "resource_type": "macro", - "tags": [], - "depends_on": { - "macros": [] - }, - "description": "", - "meta": {}, - "docs": { - "show": true - }, - "patch_path": null, - "arguments": [] - } + "model.sample_dbt.customer_details": { + "raw_sql": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", + "compiled": true, + "database": "pagila", + "schema": "pagila_dev", + "fqn": ["sample_dbt", "transform", "customer_details"], + "unique_id": "model.sample_dbt.customer_details", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "transform/customer_details.sql", + "original_file_path": "models/transform/customer_details.sql", + "name": "customer_details", + "resource_type": "model", + "alias": "customer_details", + "checksum": { + "name": "sha256", + "checksum": "038ac5af10d168699b8e5dcef54752c22a31a705eed010cacbf180e9ca214ceb" + }, + "config": { + "enabled": true, + "materialized": "ephemeral", + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "vars": {}, + "quoting": {}, + "column_types": {}, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "full_refresh": null + }, + "tags": [], + "refs": [], + "sources": [ + ["pagila", "customer"], + ["pagila", "address"], + ["pagila", "city"] + ], + "depends_on": { + "macros": [], + "nodes": [ + "source.sample_dbt.pagila.customer", + "source.sample_dbt.pagila.address", + "source.sample_dbt.pagila.city" + ] + }, + "description": "", + "columns": {}, + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "build_path": "target/compiled/sample_dbt/models/transform/customer_details.sql", + "deferred": false, + "unrendered_config": { + "materialized": "ephemeral" + }, + "compiled_sql": "\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id", + "extra_ctes_injected": true, + "extra_ctes": [], + "relation_name": null }, - "docs": { - "dbt.__overview__": { - "unique_id": "dbt.__overview__", - "package_name": "dbt", - "root_path": "/home/garylucas/dev/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", - "path": "overview.md", - "original_file_path": "docs/overview.md", - "name": "__overview__", - "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/overview)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [chat](https://community.getdbt.com/) on Slack for live questions and support." + "model.sample_dbt.monthly_billing_with_cust": { + "raw_sql": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", + "compiled": true, + "database": "pagila", + "schema": "pagila_dev", + "fqn": ["sample_dbt", "billing", "monthly_billing_with_cust"], + "unique_id": "model.sample_dbt.monthly_billing_with_cust", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "billing/monthly_billing_with_cust.sql", + "original_file_path": "models/billing/monthly_billing_with_cust.sql", + "name": "monthly_billing_with_cust", + "resource_type": "model", + "alias": "monthly_billing_with_cust", + "checksum": { + "name": "sha256", + "checksum": "96468f0564c87d3dfb75f80f0bf1dac169f67137dd46eeefdb2a9939a03fad13" + }, + "config": { + "enabled": true, + "materialized": "table", + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "vars": {}, + "quoting": {}, + "column_types": {}, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "full_refresh": null + }, + "tags": [], + "refs": [["payments_by_customer_by_month"], ["customer_details"]], + "sources": [], + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.persist_docs", + "macro.dbt.drop_relation_if_exists" + ], + "nodes": [ + "model.sample_dbt.payments_by_customer_by_month", + "model.sample_dbt.customer_details" + ] + }, + "description": "", + "columns": {}, + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "build_path": "target/run/sample_dbt/models/billing/monthly_billing_with_cust.sql", + "deferred": false, + "unrendered_config": { + "materialized": "table" + }, + "compiled_sql": "\n\nwith __dbt__CTE__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)SELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n \"pagila\".\"pagila_dev\".\"payments_by_customer_by_month\" pbc\n left outer join __dbt__CTE__customer_details cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month", + "extra_ctes_injected": true, + "extra_ctes": [ + { + "id": "model.sample_dbt.customer_details", + "sql": " __dbt__CTE__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)" } + ], + "relation_name": "\"pagila\".\"pagila_dev\".\"monthly_billing_with_cust\"" + } + }, + "sources": { + "source.sample_dbt.pagila.payment_p2020_01": { + "fqn": ["sample_dbt", "pagila", "payment_p2020_01"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "payment_p2020_01", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "payment_p2020_01", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_01\"" }, - "exposures": {}, - "selectors": {}, - "disabled": [], - "parent_map": { - "model.sample_dbt.payments_base": [ - "source.sample_dbt.pagila.payment_p2020_01", - "source.sample_dbt.pagila.payment_p2020_02", - "source.sample_dbt.pagila.payment_p2020_03", - "source.sample_dbt.pagila.payment_p2020_04", - "source.sample_dbt.pagila.payment_p2020_05", - "source.sample_dbt.pagila.payment_p2020_06" - ], - "model.sample_dbt.payments_by_customer_by_month": [ - "model.sample_dbt.payments_base" - ], - "model.sample_dbt.customer_details": [ - "source.sample_dbt.pagila.address", - "source.sample_dbt.pagila.city", - "source.sample_dbt.pagila.customer" - ], - "model.sample_dbt.monthly_billing_with_cust": [ - "model.sample_dbt.customer_details", - "model.sample_dbt.payments_by_customer_by_month" - ], - "source.sample_dbt.pagila.payment_p2020_01": [], - "source.sample_dbt.pagila.payment_p2020_02": [], - "source.sample_dbt.pagila.payment_p2020_03": [], - "source.sample_dbt.pagila.payment_p2020_04": [], - "source.sample_dbt.pagila.payment_p2020_05": [], - "source.sample_dbt.pagila.payment_p2020_06": [], - "source.sample_dbt.pagila.customer": [], - "source.sample_dbt.pagila.address": [], - "source.sample_dbt.pagila.city": [] - }, - "child_map": { - "model.sample_dbt.payments_base": [ - "model.sample_dbt.payments_by_customer_by_month" - ], - "model.sample_dbt.payments_by_customer_by_month": [ - "model.sample_dbt.monthly_billing_with_cust" - ], - "model.sample_dbt.customer_details": [ - "model.sample_dbt.monthly_billing_with_cust" - ], - "model.sample_dbt.monthly_billing_with_cust": [], - "source.sample_dbt.pagila.payment_p2020_01": [ - "model.sample_dbt.payments_base" - ], - "source.sample_dbt.pagila.payment_p2020_02": [ - "model.sample_dbt.payments_base" - ], - "source.sample_dbt.pagila.payment_p2020_03": [ - "model.sample_dbt.payments_base" - ], - "source.sample_dbt.pagila.payment_p2020_04": [ - "model.sample_dbt.payments_base" - ], - "source.sample_dbt.pagila.payment_p2020_05": [ - "model.sample_dbt.payments_base" - ], - "source.sample_dbt.pagila.payment_p2020_06": [ - "model.sample_dbt.payments_base" - ], - "source.sample_dbt.pagila.customer": [ - "model.sample_dbt.customer_details" - ], - "source.sample_dbt.pagila.address": [ - "model.sample_dbt.customer_details" - ], - "source.sample_dbt.pagila.city": [ - "model.sample_dbt.customer_details" + "source.sample_dbt.pagila.payment_p2020_02": { + "fqn": ["sample_dbt", "pagila", "payment_p2020_02"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "payment_p2020_02", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "payment_p2020_02", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_02\"" + }, + "source.sample_dbt.pagila.payment_p2020_03": { + "fqn": ["sample_dbt", "pagila", "payment_p2020_03"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "payment_p2020_03", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "payment_p2020_03", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_03\"" + }, + "source.sample_dbt.pagila.payment_p2020_04": { + "fqn": ["sample_dbt", "pagila", "payment_p2020_04"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "payment_p2020_04", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "payment_p2020_04", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_04\"" + }, + "source.sample_dbt.pagila.payment_p2020_05": { + "fqn": ["sample_dbt", "pagila", "payment_p2020_05"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "payment_p2020_05", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "payment_p2020_05", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_05\"" + }, + "source.sample_dbt.pagila.payment_p2020_06": { + "fqn": ["sample_dbt", "pagila", "payment_p2020_06"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "payment_p2020_06", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "payment_p2020_06", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_06\"" + }, + "source.sample_dbt.pagila.customer": { + "fqn": ["sample_dbt", "pagila", "customer"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.customer", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "customer", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "customer", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"customer\"" + }, + "source.sample_dbt.pagila.address": { + "fqn": ["sample_dbt", "pagila", "address"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.address", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "address", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "address", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"address\"" + }, + "source.sample_dbt.pagila.city": { + "fqn": ["sample_dbt", "pagila", "city"], + "database": "pagila", + "schema": "public", + "unique_id": "source.sample_dbt.pagila.city", + "package_name": "sample_dbt", + "root_path": "/datahub-contrib/sample-dbt", + "path": "models/base.yml", + "original_file_path": "models/base.yml", + "name": "city", + "source_name": "pagila", + "source_description": "base models", + "loader": "", + "identifier": "city", + "resource_type": "source", + "quoting": { + "database": null, + "schema": null, + "identifier": null, + "column": null + }, + "loaded_at_field": null, + "freshness": { + "warn_after": null, + "error_after": null, + "filter": null + }, + "external": null, + "description": "", + "columns": {}, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": { + "enabled": true + }, + "patch_path": null, + "unrendered_config": {}, + "relation_name": "\"pagila\".\"public\".\"city\"" + } + }, + "macros": { + "macro.dbt_postgres.postgres_get_relations": { + "unique_id": "macro.dbt_postgres.postgres_get_relations", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/relations.sql", + "original_file_path": "macros/relations.sql", + "name": "postgres_get_relations", + "macro_sql": "{% macro postgres_get_relations () -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": ["macro.dbt.statement"] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__get_catalog": { + "unique_id": "macro.dbt_postgres.postgres__get_catalog", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/catalog.sql", + "original_file_path": "macros/catalog.sql", + "name": "postgres__get_catalog", + "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n\n {%- call statement('catalog', fetch_result=True) -%}\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n\n where (\n {%- for schema in schemas -%}\n upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence = 'p' -- [p]ermanent table. Other values are [u]nlogged table, [t]emporary table\n and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__create_table_as": { + "unique_id": "macro.dbt_postgres.postgres__create_table_as", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__create_table_as", + "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__create_schema": { + "unique_id": "macro.dbt_postgres.postgres__create_schema", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__create_schema", + "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__drop_schema": { + "unique_id": "macro.dbt_postgres.postgres__drop_schema", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__drop_schema", + "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__get_columns_in_relation": { + "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__get_columns_in_relation", + "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__list_relations_without_caching": { + "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__list_relations_without_caching", + "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__information_schema_name": { + "unique_id": "macro.dbt_postgres.postgres__information_schema_name", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__information_schema_name", + "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__list_schemas": { + "unique_id": "macro.dbt_postgres.postgres__list_schemas", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__list_schemas", + "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__check_schema_exists": { + "unique_id": "macro.dbt_postgres.postgres__check_schema_exists", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__check_schema_exists", + "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__current_timestamp": { + "unique_id": "macro.dbt_postgres.postgres__current_timestamp", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__current_timestamp", + "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__snapshot_string_as_time": { + "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__snapshot_string_as_time", + "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__snapshot_get_time": { + "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__snapshot_get_time", + "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__make_temp_relation": { + "unique_id": "macro.dbt_postgres.postgres__make_temp_relation", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__make_temp_relation", + "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix_length = suffix|length + dtstring|length %}\n {% set relation_max_name_length = 63 %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}\n {% endif %}\n {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}\n {% do return(base_relation.incorporate(\n path={\n \"identifier\": tmp_identifier,\n \"schema\": none,\n \"database\": none\n })) -%}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres_escape_comment": { + "unique_id": "macro.dbt_postgres.postgres_escape_comment", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres_escape_comment", + "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__alter_relation_comment": { + "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__alter_relation_comment", + "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__alter_column_comment": { + "unique_id": "macro.dbt_postgres.postgres__alter_column_comment", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/adapters.sql", + "original_file_path": "macros/adapters.sql", + "name": "postgres__alter_column_comment", + "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt_postgres.postgres__snapshot_merge_sql": { + "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql", + "package_name": "dbt_postgres", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/postgres", + "path": "macros/materializations/snapshot_merge.sql", + "original_file_path": "macros/materializations/snapshot_merge.sql", + "name": "postgres__snapshot_merge_sql", + "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.statement": { + "unique_id": "macro.dbt.statement", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/core.sql", + "original_file_path": "macros/core.sql", + "name": "statement", + "macro_sql": "{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.noop_statement": { + "unique_id": "macro.dbt.noop_statement", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/core.sql", + "original_file_path": "macros/core.sql", + "name": "noop_statement", + "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.run_hooks": { + "unique_id": "macro.dbt.run_hooks", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "run_hooks", + "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.column_list": { + "unique_id": "macro.dbt.column_list", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "column_list", + "macro_sql": "{% macro column_list(columns) %}\n {%- for col in columns %}\n {{ col.name }} {% if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.column_list_for_create_table": { + "unique_id": "macro.dbt.column_list_for_create_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "column_list_for_create_table", + "macro_sql": "{% macro column_list_for_create_table(columns) %}\n {%- for col in columns %}\n {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %}\n {% endfor -%}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.make_hook_config": { + "unique_id": "macro.dbt.make_hook_config", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "make_hook_config", + "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.before_begin": { + "unique_id": "macro.dbt.before_begin", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "before_begin", + "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.in_transaction": { + "unique_id": "macro.dbt.in_transaction", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "in_transaction", + "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.after_commit": { + "unique_id": "macro.dbt.after_commit", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "after_commit", + "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.drop_relation_if_exists": { + "unique_id": "macro.dbt.drop_relation_if_exists", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "drop_relation_if_exists", + "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.load_relation": { + "unique_id": "macro.dbt.load_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "load_relation", + "macro_sql": "{% macro load_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.should_full_refresh": { + "unique_id": "macro.dbt.should_full_refresh", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/helpers.sql", + "original_file_path": "macros/materializations/helpers.sql", + "name": "should_full_refresh", + "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_merge_sql": { + "unique_id": "macro.dbt.get_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "get_merge_sql", + "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}\n {{ adapter.dispatch('get_merge_sql')(target, source, unique_key, dest_columns, predicates) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_delete_insert_merge_sql": { + "unique_id": "macro.dbt.get_delete_insert_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "get_delete_insert_merge_sql", + "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql')(target, source, unique_key, dest_columns) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_insert_overwrite_merge_sql": { + "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "get_insert_overwrite_merge_sql", + "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__get_merge_sql": { + "unique_id": "macro.dbt.default__get_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "default__get_merge_sql", + "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n {% if unique_key %}\n when matched then update set\n {% for column in dest_columns -%}\n {{ adapter.quote(column.name) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(column.name) }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_quoted_csv": { + "unique_id": "macro.dbt.get_quoted_csv", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "get_quoted_csv", + "macro_sql": "{% macro get_quoted_csv(column_names) %}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.common_get_delete_insert_merge_sql": { + "unique_id": "macro.dbt.common_get_delete_insert_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "common_get_delete_insert_merge_sql", + "macro_sql": "{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key is not none %}\n delete from {{ target }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n );\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n );\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__get_delete_insert_merge_sql": { + "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "default__get_delete_insert_merge_sql", + "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}\n {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__get_insert_overwrite_merge_sql": { + "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/common/merge.sql", + "original_file_path": "macros/materializations/common/merge.sql", + "name": "default__get_insert_overwrite_merge_sql", + "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.materialization_table_default": { + "unique_id": "macro.dbt.materialization_table_default", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/table/table.sql", + "original_file_path": "macros/materializations/table/table.sql", + "name": "materialization_table_default", + "macro_sql": "{% materialization table, default %}\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema,\n database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_table_as(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.create_csv_table": { + "unique_id": "macro.dbt.create_csv_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "create_csv_table", + "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table')(model, agate_table) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.reset_csv_table": { + "unique_id": "macro.dbt.reset_csv_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "reset_csv_table", + "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.load_csv_rows": { + "unique_id": "macro.dbt.load_csv_rows", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "load_csv_rows", + "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__create_csv_table": { + "unique_id": "macro.dbt.default__create_csv_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "default__create_csv_table", + "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__reset_csv_table": { + "unique_id": "macro.dbt.default__reset_csv_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "default__reset_csv_table", + "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_seed_column_quoted_csv": { + "unique_id": "macro.dbt.get_seed_column_quoted_csv", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "get_seed_column_quoted_csv", + "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.basic_load_csv_rows": { + "unique_id": "macro.dbt.basic_load_csv_rows", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "basic_load_csv_rows", + "macro_sql": "{% macro basic_load_csv_rows(model, batch_size, agate_table) %}\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n %s\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__load_csv_rows": { + "unique_id": "macro.dbt.default__load_csv_rows", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "default__load_csv_rows", + "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.materialization_seed_default": { + "unique_id": "macro.dbt.materialization_seed_default", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/seed/seed.sql", + "original_file_path": "macros/materializations/seed/seed.sql", + "name": "materialization_seed_default", + "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set agate_table = load_agate_table() -%}\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ create_table_sql }};\n -- dbt seed --\n {{ sql }}\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.materialization_view_default": { + "unique_id": "macro.dbt.materialization_view_default", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/view/view.sql", + "original_file_path": "macros/materializations/view/view.sql", + "name": "materialization_view_default", + "macro_sql": "{%- materialization view, default -%}\n\n {%- set identifier = model['alias'] -%}\n {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}\n {%- set backup_identifier = model['name'] + '__dbt_backup' -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,\n schema=schema, database=database, type='view') -%}\n\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"old_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the old_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the old_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}\n {%- set backup_relation = api.Relation.create(identifier=backup_identifier,\n schema=schema, database=database,\n type=backup_relation_type) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exists for some reason\n {{ adapter.drop_relation(intermediate_relation) }}\n {{ adapter.drop_relation(backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if old_relation is not none %}\n {{ adapter.rename_relation(target_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.handle_existing_table": { + "unique_id": "macro.dbt.handle_existing_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/view/create_or_replace_view.sql", + "original_file_path": "macros/materializations/view/create_or_replace_view.sql", + "name": "handle_existing_table", + "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch(\"handle_existing_table\", packages=['dbt'])(full_refresh, old_relation) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__handle_existing_table": { + "unique_id": "macro.dbt.default__handle_existing_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/view/create_or_replace_view.sql", + "original_file_path": "macros/materializations/view/create_or_replace_view.sql", + "name": "default__handle_existing_table", + "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.create_or_replace_view": { + "unique_id": "macro.dbt.create_or_replace_view", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/view/create_or_replace_view.sql", + "original_file_path": "macros/materializations/view/create_or_replace_view.sql", + "name": "create_or_replace_view", + "macro_sql": "{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n\n {% if run_outside_transaction_hooks %}\n -- no transactions on BigQuery\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n {% endif %}\n\n -- `BEGIN` happens here on Snowflake\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ create_view_as(target_relation, sql) }}\n {%- endcall %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if run_outside_transaction_hooks %}\n -- No transactions on BigQuery\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.materialization_incremental_default": { + "unique_id": "macro.dbt.materialization_incremental_default", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/incremental/incremental.sql", + "original_file_path": "macros/materializations/incremental/incremental.sql", + "name": "materialization_incremental_default", + "macro_sql": "{% materialization incremental, default -%}\n\n {% set unique_key = config.get('unique_key') %}\n\n {% set target_relation = this.incorporate(type='table') %}\n {% set existing_relation = load_relation(this) %}\n {% set tmp_relation = make_temp_relation(this) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n {% if existing_relation is none %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% elif existing_relation.is_view or should_full_refresh() %}\n {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}\n {% set backup_identifier = existing_relation.identifier ~ \"__dbt_backup\" %}\n {% set backup_relation = existing_relation.incorporate(path={\"identifier\": backup_identifier}) %}\n {% do adapter.drop_relation(backup_relation) %}\n\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% set build_sql = create_table_as(False, target_relation, sql) %}\n {% do to_drop.append(backup_relation) %}\n {% else %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n {% do run_query(create_table_as(True, tmp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=tmp_relation,\n to_relation=target_relation) %}\n {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.incremental_upsert": { + "unique_id": "macro.dbt.incremental_upsert", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/incremental/helpers.sql", + "original_file_path": "macros/materializations/incremental/helpers.sql", + "name": "incremental_upsert", + "macro_sql": "{% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name=\"main\") %}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n\n {%- if unique_key is not none -%}\n delete\n from {{ target_relation }}\n where ({{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ tmp_relation }}\n );\n {%- endif %}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ tmp_relation }}\n );\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.create_columns": { + "unique_id": "macro.dbt.create_columns", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "create_columns", + "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns')(relation, columns) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__create_columns": { + "unique_id": "macro.dbt.default__create_columns", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "default__create_columns", + "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.post_snapshot": { + "unique_id": "macro.dbt.post_snapshot", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "post_snapshot", + "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot')(staging_relation) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__post_snapshot": { + "unique_id": "macro.dbt.default__post_snapshot", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "default__post_snapshot", + "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_staging_table": { + "unique_id": "macro.dbt.snapshot_staging_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "snapshot_staging_table", + "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select \n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n \n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n \n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.build_snapshot_table": { + "unique_id": "macro.dbt.build_snapshot_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "build_snapshot_table", + "macro_sql": "{% macro build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_or_create_relation": { + "unique_id": "macro.dbt.get_or_create_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "get_or_create_relation", + "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.build_snapshot_staging_table": { + "unique_id": "macro.dbt.build_snapshot_staging_table", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "build_snapshot_staging_table", + "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.materialization_snapshot_default": { + "unique_id": "macro.dbt.materialization_snapshot_default", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot.sql", + "original_file_path": "macros/materializations/snapshot/snapshot.sql", + "name": "materialization_snapshot_default", + "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.database, model.schema) %}\n {% endif %}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.strategy_dispatch": { + "unique_id": "macro.dbt.strategy_dispatch", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "strategy_dispatch", + "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_hash_arguments": { + "unique_id": "macro.dbt.snapshot_hash_arguments", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "snapshot_hash_arguments", + "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments')(args) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__snapshot_hash_arguments": { + "unique_id": "macro.dbt.default__snapshot_hash_arguments", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "default__snapshot_hash_arguments", + "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_get_time": { + "unique_id": "macro.dbt.snapshot_get_time", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "snapshot_get_time", + "macro_sql": "{% macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time')() }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__snapshot_get_time": { + "unique_id": "macro.dbt.default__snapshot_get_time", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "default__snapshot_get_time", + "macro_sql": "{% macro default__snapshot_get_time() -%}\n {{ current_timestamp() }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_timestamp_strategy": { + "unique_id": "macro.dbt.snapshot_timestamp_strategy", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "snapshot_timestamp_strategy", + "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/fishtown-analytics/dbt/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_string_as_time": { + "unique_id": "macro.dbt.snapshot_string_as_time", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "snapshot_string_as_time", + "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time')(timestamp) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__snapshot_string_as_time": { + "unique_id": "macro.dbt.default__snapshot_string_as_time", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "default__snapshot_string_as_time", + "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_check_all_get_existing_columns": { + "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "snapshot_check_all_get_existing_columns", + "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}\n {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}\n {%- if not target_exists -%}\n {# no table yet -> return whatever the query does #}\n {{ return([false, query_columns]) }}\n {%- endif -%}\n {# handle any schema changes #}\n {%- set target_table = node.get('alias', node.get('name')) -%}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=target_table) -%}\n {%- set existing_cols = get_columns_in_query('select * from ' ~ target_relation) -%}\n {%- set ns = namespace() -%} {# handle for-loop scoping with a namespace #}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(col) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return([ns.column_added, intersection]) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_check_strategy": { + "unique_id": "macro.dbt.snapshot_check_strategy", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/strategies.sql", + "original_file_path": "macros/materializations/snapshot/strategies.sql", + "name": "snapshot_check_strategy", + "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n \n {% set select_current_time -%}\n select {{ snapshot_get_time() }} as snapshot_start\n {%- endset %}\n\n {#-- don't access the column by name, to avoid dealing with casing issues on snowflake #}\n {%- set now = run_query(select_current_time)[0][0] -%}\n {% if now is none or now is undefined -%}\n {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}\n {%- endif %}\n {% set updated_at = snapshot_string_as_time(now) %}\n\n {% set column_added = false %}\n\n {% if check_cols_config == 'all' %}\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists) %}\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {% set check_cols = check_cols_config %}\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n TRUE\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.snapshot_merge_sql": { + "unique_id": "macro.dbt.snapshot_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot_merge.sql", + "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", + "name": "snapshot_merge_sql", + "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql')(target, source, insert_cols) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__snapshot_merge_sql": { + "unique_id": "macro.dbt.default__snapshot_merge_sql", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/materializations/snapshot/snapshot_merge.sql", + "original_file_path": "macros/materializations/snapshot/snapshot_merge.sql", + "name": "default__snapshot_merge_sql", + "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n ;\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_columns_in_query": { + "unique_id": "macro.dbt.get_columns_in_query", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "get_columns_in_query", + "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__get_columns_in_query": { + "unique_id": "macro.dbt.default__get_columns_in_query", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__get_columns_in_query", + "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n {% endcall %}\n\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.create_schema": { + "unique_id": "macro.dbt.create_schema", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "create_schema", + "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema')(relation) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": ["macro.dbt_postgres.postgres__create_schema"] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__create_schema": { + "unique_id": "macro.dbt.default__create_schema", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__create_schema", + "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.drop_schema": { + "unique_id": "macro.dbt.drop_schema", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "drop_schema", + "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema')(relation) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__drop_schema": { + "unique_id": "macro.dbt.default__drop_schema", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__drop_schema", + "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.create_table_as": { + "unique_id": "macro.dbt.create_table_as", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "create_table_as", + "macro_sql": "{% macro create_table_as(temporary, relation, sql) -%}\n {{ adapter.dispatch('create_table_as')(temporary, relation, sql) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__create_table_as": { + "unique_id": "macro.dbt.default__create_table_as", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__create_table_as", + "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n as (\n {{ sql }}\n );\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.create_view_as": { + "unique_id": "macro.dbt.create_view_as", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "create_view_as", + "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as')(relation, sql) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__create_view_as": { + "unique_id": "macro.dbt.default__create_view_as", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__create_view_as", + "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }} as (\n {{ sql }}\n );\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_catalog": { + "unique_id": "macro.dbt.get_catalog", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "get_catalog", + "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog')(information_schema, schemas)) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__get_catalog": { + "unique_id": "macro.dbt.default__get_catalog", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__get_catalog", + "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.get_columns_in_relation": { + "unique_id": "macro.dbt.get_columns_in_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "get_columns_in_relation", + "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation')(relation)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.sql_convert_columns_in_relation": { + "unique_id": "macro.dbt.sql_convert_columns_in_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "sql_convert_columns_in_relation", + "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__get_columns_in_relation": { + "unique_id": "macro.dbt.default__get_columns_in_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__get_columns_in_relation", + "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.alter_column_type": { + "unique_id": "macro.dbt.alter_column_type", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "alter_column_type", + "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type')(relation, column_name, new_column_type)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.alter_column_comment": { + "unique_id": "macro.dbt.alter_column_comment", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "alter_column_comment", + "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment')(relation, column_dict)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__alter_column_comment": { + "unique_id": "macro.dbt.default__alter_column_comment", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__alter_column_comment", + "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.alter_relation_comment": { + "unique_id": "macro.dbt.alter_relation_comment", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "alter_relation_comment", + "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment')(relation, relation_comment)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__alter_relation_comment": { + "unique_id": "macro.dbt.default__alter_relation_comment", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__alter_relation_comment", + "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.persist_docs": { + "unique_id": "macro.dbt.persist_docs", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "persist_docs", + "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__persist_docs": { + "unique_id": "macro.dbt.default__persist_docs", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__persist_docs", + "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__alter_column_type": { + "unique_id": "macro.dbt.default__alter_column_type", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__alter_column_type", + "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.drop_relation": { + "unique_id": "macro.dbt.drop_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "drop_relation", + "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation')(relation)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": ["macro.dbt.default__drop_relation"] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__drop_relation": { + "unique_id": "macro.dbt.default__drop_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__drop_relation", + "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.truncate_relation": { + "unique_id": "macro.dbt.truncate_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "truncate_relation", + "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation')(relation)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__truncate_relation": { + "unique_id": "macro.dbt.default__truncate_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__truncate_relation", + "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.rename_relation": { + "unique_id": "macro.dbt.rename_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "rename_relation", + "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation')(from_relation, to_relation)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": ["macro.dbt.default__rename_relation"] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__rename_relation": { + "unique_id": "macro.dbt.default__rename_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__rename_relation", + "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.information_schema_name": { + "unique_id": "macro.dbt.information_schema_name", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "information_schema_name", + "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name')(database)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__information_schema_name": { + "unique_id": "macro.dbt.default__information_schema_name", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__information_schema_name", + "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.list_schemas": { + "unique_id": "macro.dbt.list_schemas", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "list_schemas", + "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas')(database)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": ["macro.dbt_postgres.postgres__list_schemas"] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__list_schemas": { + "unique_id": "macro.dbt.default__list_schemas", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__list_schemas", + "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.check_schema_exists": { + "unique_id": "macro.dbt.check_schema_exists", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "check_schema_exists", + "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists')(information_schema, schema)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__check_schema_exists": { + "unique_id": "macro.dbt.default__check_schema_exists", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__check_schema_exists", + "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.list_relations_without_caching": { + "unique_id": "macro.dbt.list_relations_without_caching", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "list_relations_without_caching", + "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching')(schema_relation)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_relations_without_caching" ] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__list_relations_without_caching": { + "unique_id": "macro.dbt.default__list_relations_without_caching", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__list_relations_without_caching", + "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.current_timestamp": { + "unique_id": "macro.dbt.current_timestamp", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "current_timestamp", + "macro_sql": "{% macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp')() }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__current_timestamp": { + "unique_id": "macro.dbt.default__current_timestamp", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__current_timestamp", + "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter '+adapter.type()) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.collect_freshness": { + "unique_id": "macro.dbt.collect_freshness", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "collect_freshness", + "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness')(source, loaded_at_field, filter))}}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__collect_freshness": { + "unique_id": "macro.dbt.default__collect_freshness", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__collect_freshness", + "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness').table) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.make_temp_relation": { + "unique_id": "macro.dbt.make_temp_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "make_temp_relation", + "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation')(base_relation, suffix))}}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__make_temp_relation": { + "unique_id": "macro.dbt.default__make_temp_relation", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "default__make_temp_relation", + "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(\n path={\"identifier\": tmp_identifier}) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.set_sql_header": { + "unique_id": "macro.dbt.set_sql_header", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/adapters/common.sql", + "original_file_path": "macros/adapters/common.sql", + "name": "set_sql_header", + "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__test_relationships": { + "unique_id": "macro.dbt.default__test_relationships", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/relationships.sql", + "original_file_path": "macros/schema_tests/relationships.sql", + "name": "default__test_relationships", + "macro_sql": "{% macro default__test_relationships(model, to, field) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}\n\n\nselect count(*) as validation_errors\nfrom (\n select {{ column_name }} as id from {{ model }}\n) as child\nleft join (\n select {{ field }} as id from {{ to }}\n) as parent on parent.id = child.id\nwhere child.id is not null\n and parent.id is null\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.test_relationships": { + "unique_id": "macro.dbt.test_relationships", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/relationships.sql", + "original_file_path": "macros/schema_tests/relationships.sql", + "name": "test_relationships", + "macro_sql": "{% macro test_relationships(model, to, field) %}\n {% set macro = adapter.dispatch('test_relationships') %}\n {{ macro(model, to, field, **kwargs) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__test_not_null": { + "unique_id": "macro.dbt.default__test_not_null", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/not_null.sql", + "original_file_path": "macros/schema_tests/not_null.sql", + "name": "default__test_not_null", + "macro_sql": "{% macro default__test_not_null(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n\nselect count(*) as validation_errors\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.test_not_null": { + "unique_id": "macro.dbt.test_not_null", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/not_null.sql", + "original_file_path": "macros/schema_tests/not_null.sql", + "name": "test_not_null", + "macro_sql": "{% macro test_not_null(model) %}\n {% set macro = adapter.dispatch('test_not_null') %}\n {{ macro(model, **kwargs) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__test_unique": { + "unique_id": "macro.dbt.default__test_unique", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/unique.sql", + "original_file_path": "macros/schema_tests/unique.sql", + "name": "default__test_unique", + "macro_sql": "{% macro default__test_unique(model) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}\n\nselect count(*) as validation_errors\nfrom (\n\n select\n {{ column_name }}\n\n from {{ model }}\n where {{ column_name }} is not null\n group by {{ column_name }}\n having count(*) > 1\n\n) validation_errors\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.test_unique": { + "unique_id": "macro.dbt.test_unique", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/unique.sql", + "original_file_path": "macros/schema_tests/unique.sql", + "name": "test_unique", + "macro_sql": "{% macro test_unique(model) %}\n {% set macro = adapter.dispatch('test_unique') %}\n {{ macro(model, **kwargs) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__test_accepted_values": { + "unique_id": "macro.dbt.default__test_accepted_values", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/accepted_values.sql", + "original_file_path": "macros/schema_tests/accepted_values.sql", + "name": "default__test_accepted_values", + "macro_sql": "{% macro default__test_accepted_values(model, values) %}\n\n{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}\n{% set quote_values = kwargs.get('quote', True) %}\n\nwith all_values as (\n\n select distinct\n {{ column_name }} as value_field\n\n from {{ model }}\n\n),\n\nvalidation_errors as (\n\n select\n value_field\n\n from all_values\n where value_field not in (\n {% for value in values -%}\n {% if quote_values -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n )\n)\n\nselect count(*) as validation_errors\nfrom validation_errors\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.test_accepted_values": { + "unique_id": "macro.dbt.test_accepted_values", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/schema_tests/accepted_values.sql", + "original_file_path": "macros/schema_tests/accepted_values.sql", + "name": "test_accepted_values", + "macro_sql": "{% macro test_accepted_values(model, values) %}\n {% set macro = adapter.dispatch('test_accepted_values') %}\n {{ macro(model, values, **kwargs) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.is_incremental": { + "unique_id": "macro.dbt.is_incremental", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/is_incremental.sql", + "original_file_path": "macros/etc/is_incremental.sql", + "name": "is_incremental", + "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.run_query": { + "unique_id": "macro.dbt.run_query", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/query.sql", + "original_file_path": "macros/etc/query.sql", + "name": "run_query", + "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.generate_database_name": { + "unique_id": "macro.dbt.generate_database_name", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/get_custom_database.sql", + "original_file_path": "macros/etc/get_custom_database.sql", + "name": "generate_database_name", + "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name')(custom_database_name, node)) %}\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": ["macro.dbt.default__generate_database_name"] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.default__generate_database_name": { + "unique_id": "macro.dbt.default__generate_database_name", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/get_custom_database.sql", + "original_file_path": "macros/etc/get_custom_database.sql", + "name": "default__generate_database_name", + "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.generate_alias_name": { + "unique_id": "macro.dbt.generate_alias_name", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/get_custom_alias.sql", + "original_file_path": "macros/etc/get_custom_alias.sql", + "name": "generate_alias_name", + "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name is none -%}\n\n {{ node.name }}\n\n {%- else -%}\n\n {{ custom_alias_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.generate_schema_name": { + "unique_id": "macro.dbt.generate_schema_name", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/get_custom_schema.sql", + "original_file_path": "macros/etc/get_custom_schema.sql", + "name": "generate_schema_name", + "macro_sql": "{% macro generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.generate_schema_name_for_env": { + "unique_id": "macro.dbt.generate_schema_name_for_env", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/get_custom_schema.sql", + "original_file_path": "macros/etc/get_custom_schema.sql", + "name": "generate_schema_name_for_env", + "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.convert_datetime": { + "unique_id": "macro.dbt.convert_datetime", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/datetime.sql", + "original_file_path": "macros/etc/datetime.sql", + "name": "convert_datetime", + "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.dates_in_range": { + "unique_id": "macro.dbt.dates_in_range", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/datetime.sql", + "original_file_path": "macros/etc/datetime.sql", + "name": "dates_in_range", + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partiton start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.partition_range": { + "unique_id": "macro.dbt.partition_range", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/datetime.sql", + "original_file_path": "macros/etc/datetime.sql", + "name": "partition_range", + "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + }, + "macro.dbt.py_current_timestring": { + "unique_id": "macro.dbt.py_current_timestring", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "macros/etc/datetime.sql", + "original_file_path": "macros/etc/datetime.sql", + "name": "py_current_timestring", + "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", + "resource_type": "macro", + "tags": [], + "depends_on": { + "macros": [] + }, + "description": "", + "meta": {}, + "docs": { + "show": true + }, + "patch_path": null, + "arguments": [] + } + }, + "docs": { + "dbt.__overview__": { + "unique_id": "dbt.__overview__", + "package_name": "dbt", + "root_path": "/datahub-contrib/sample-dbt/dbt-env/lib/python3.8/site-packages/dbt/include/global_project", + "path": "overview.md", + "original_file_path": "docs/overview.md", + "name": "__overview__", + "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/overview)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [chat](https://community.getdbt.com/) on Slack for live questions and support." } -} \ No newline at end of file + }, + "exposures": {}, + "selectors": {}, + "disabled": [], + "parent_map": { + "model.sample_dbt.payments_base": [ + "source.sample_dbt.pagila.payment_p2020_01", + "source.sample_dbt.pagila.payment_p2020_02", + "source.sample_dbt.pagila.payment_p2020_03", + "source.sample_dbt.pagila.payment_p2020_04", + "source.sample_dbt.pagila.payment_p2020_05", + "source.sample_dbt.pagila.payment_p2020_06" + ], + "model.sample_dbt.payments_by_customer_by_month": [ + "model.sample_dbt.payments_base" + ], + "model.sample_dbt.customer_details": [ + "source.sample_dbt.pagila.address", + "source.sample_dbt.pagila.city", + "source.sample_dbt.pagila.customer" + ], + "model.sample_dbt.monthly_billing_with_cust": [ + "model.sample_dbt.customer_details", + "model.sample_dbt.payments_by_customer_by_month" + ], + "source.sample_dbt.pagila.payment_p2020_01": [], + "source.sample_dbt.pagila.payment_p2020_02": [], + "source.sample_dbt.pagila.payment_p2020_03": [], + "source.sample_dbt.pagila.payment_p2020_04": [], + "source.sample_dbt.pagila.payment_p2020_05": [], + "source.sample_dbt.pagila.payment_p2020_06": [], + "source.sample_dbt.pagila.customer": [], + "source.sample_dbt.pagila.address": [], + "source.sample_dbt.pagila.city": [] + }, + "child_map": { + "model.sample_dbt.payments_base": [ + "model.sample_dbt.payments_by_customer_by_month" + ], + "model.sample_dbt.payments_by_customer_by_month": [ + "model.sample_dbt.monthly_billing_with_cust" + ], + "model.sample_dbt.customer_details": [ + "model.sample_dbt.monthly_billing_with_cust" + ], + "model.sample_dbt.monthly_billing_with_cust": [], + "source.sample_dbt.pagila.payment_p2020_01": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_02": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_03": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_04": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_05": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_06": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.customer": ["model.sample_dbt.customer_details"], + "source.sample_dbt.pagila.address": ["model.sample_dbt.customer_details"], + "source.sample_dbt.pagila.city": ["model.sample_dbt.customer_details"] + } +} diff --git a/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json index b8322977ddef17..685f09195444a4 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json @@ -185,7 +185,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -269,7 +269,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -306,6 +306,21 @@ "recursive": false, "globalTags": null, "glossaryTerms": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "character varying(129)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null } ], "primaryKeys": null, @@ -478,7 +493,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -530,6 +545,53 @@ "recursive": false, "globalTags": null, "glossaryTerms": null + }, + { + "fieldPath": "emails", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "text[]", + "recursive": false, + "globalTags": null, + "glossaryTerms": null + }, + { + "fieldPath": "cost", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "money", + "recursive": false, + "globalTags": null, + "glossaryTerms": null + }, + { + "fieldPath": "native", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "some native type", + "recursive": false, + "globalTags": null, + "glossaryTerms": null } ], "primaryKeys": null, @@ -672,7 +734,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -821,7 +883,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -970,7 +1032,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -1119,7 +1181,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -1268,7 +1330,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -1417,7 +1479,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -1611,7 +1673,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -1805,7 +1867,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", @@ -1924,7 +1986,7 @@ "description": null, "type": { "type": { - "com.linkedin.pegasus2avro.schema.DateType": {} + "com.linkedin.pegasus2avro.schema.TimeType": {} } }, "nativeDataType": "timestamp with time zone", diff --git a/metadata-ingestion/tests/integration/feast/feast_mce_golden.json b/metadata-ingestion/tests/integration/feast/feast_mces_golden.json similarity index 100% rename from metadata-ingestion/tests/integration/feast/feast_mce_golden.json rename to metadata-ingestion/tests/integration/feast/feast_mces_golden.json diff --git a/metadata-ingestion/tests/integration/feast/test_feast.py b/metadata-ingestion/tests/integration/feast/test_feast.py index 24eb4dc09e7176..60a3c4e39f8da4 100644 --- a/metadata-ingestion/tests/integration/feast/test_feast.py +++ b/metadata-ingestion/tests/integration/feast/test_feast.py @@ -46,6 +46,6 @@ def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path): # Verify the output. output = mce_helpers.load_json_file(str(tmp_path / "feast_mces.json")) golden = mce_helpers.load_json_file( - str(test_resources_dir / "feast_mce_golden.json") + str(test_resources_dir / "feast_mces_golden.json") ) mce_helpers.assert_mces_equal(output, golden) diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mce_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json similarity index 100% rename from metadata-ingestion/tests/integration/ldap/ldap_mce_golden.json rename to metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json diff --git a/metadata-ingestion/tests/integration/ldap/test_ldap.py b/metadata-ingestion/tests/integration/ldap/test_ldap.py index c9fc89599b4c9b..b58426c3c60132 100644 --- a/metadata-ingestion/tests/integration/ldap/test_ldap.py +++ b/metadata-ingestion/tests/integration/ldap/test_ldap.py @@ -44,6 +44,6 @@ def test_ldap_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): output = mce_helpers.load_json_file(str(tmp_path / "ldap_mces.json")) golden = mce_helpers.load_json_file( - str(test_resources_dir / "ldap_mce_golden.json") + str(test_resources_dir / "ldap_mces_golden.json") ) mce_helpers.assert_mces_equal(output, golden) diff --git a/metadata-ingestion/tests/integration/mongodb/mongodb_mce_golden.json b/metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json similarity index 100% rename from metadata-ingestion/tests/integration/mongodb/mongodb_mce_golden.json rename to metadata-ingestion/tests/integration/mongodb/mongodb_mces_golden.json diff --git a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py index 40300f9b01e444..d0f0f1f80572df 100644 --- a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py +++ b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py @@ -40,6 +40,6 @@ def test_mongodb_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time # Verify the output. output = mce_helpers.load_json_file(str(tmp_path / "mongodb_mces.json")) golden = mce_helpers.load_json_file( - str(test_resources_dir / "mongodb_mce_golden.json") + str(test_resources_dir / "mongodb_mces_golden.json") ) mce_helpers.assert_mces_equal(output, golden) diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mce_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json similarity index 100% rename from metadata-ingestion/tests/integration/mysql/mysql_mce_golden.json rename to metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json diff --git a/metadata-ingestion/tests/integration/mysql/test_mysql.py b/metadata-ingestion/tests/integration/mysql/test_mysql.py index 6cd7951aed0428..256456789a3134 100644 --- a/metadata-ingestion/tests/integration/mysql/test_mysql.py +++ b/metadata-ingestion/tests/integration/mysql/test_mysql.py @@ -26,6 +26,6 @@ def test_mysql_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): # Verify the output. golden = mce_helpers.load_json_file( - str(test_resources_dir / "mysql_mce_golden.json") + str(test_resources_dir / "mysql_mces_golden.json") ) mce_helpers.assert_mces_equal(output, golden) diff --git a/metadata-ingestion/tests/integration/sql_server/mssql_mce_golden.json b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json similarity index 100% rename from metadata-ingestion/tests/integration/sql_server/mssql_mce_golden.json rename to metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json diff --git a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py index 62d0efbb1e7792..37dfc43dbbe662 100644 --- a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py +++ b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py @@ -41,6 +41,6 @@ def test_mssql_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): # Verify the output. golden = mce_helpers.load_json_file( - str(test_resources_dir / "mssql_mce_golden.json") + str(test_resources_dir / "mssql_mces_golden.json") ) mce_helpers.assert_mces_equal(output, golden)