diff --git a/CHANGELOG.md b/CHANGELOG.md index a42154da..7d409e3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +# dbt-external-tables v0.8.0 + +This release supports any version (minor and patch) of v1, which means far less need for compatibility releases in the future. + +### Features +- (Snowflake) Support for regex `pattern` in snowpipes ([#111](https://github.com/dbt-labs/dbt-external-tables/pull/111), [#122](https://github.com/dbt-labs/dbt-external-tables/pull/122)) +- (Apache Spark) Real support for partitioned external tables. Note that external sources with `partitions` defined were implicitly skipped. Going forward, sources with partitions defined (excluding those with `using: delta`) will run `alter table ... recover partitions`. + +### Under the hood +- Use standard logging, thereby removing dependency on `dbt_utils` ([#119](https://github.com/dbt-labs/dbt-external-tables/pull/119)) +- Remove `synapse__`-prefixed "passthrough" macros, now that `dbt-synapse` can use `sqlserver__`-prefixed macros instead ([#110](https://github.com/dbt-labs/dbt-external-tables/pull/110)) + +### Contributors +- [@JCZuurmond](https://github.com/JCZuurmond) ([#116](https://github.com/dbt-labs/dbt-external-tables/pull/116)) +- [@stumelius](https://github.com/stumelius) ([#111](https://github.com/dbt-labs/dbt-external-tables/pull/111)) +- [@swanderz](https://github.com/swanderz) ([#110](https://github.com/dbt-labs/dbt-external-tables/pull/110)) + # dbt-external-tables v0.7.3 ### Fixes diff --git a/dbt_project.yml b/dbt_project.yml index 678215d9..39a7f49d 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,17 +1,5 @@ name: 'dbt_external_tables' -version: '0.7.0' - +version: '0.8.0' config-version: 2 - -require-dbt-version: [">=0.20.0", "<1.1.0"] - -source-paths: ["models"] -analysis-paths: ["analysis"] -test-paths: ["tests"] -data-paths: ["data"] +require-dbt-version: [">=1.0.0", "<2.0.0"] macro-paths: ["macros"] - -target-path: "target" -clean-targets: - - "target" - - "dbt_modules" diff --git a/integration_tests/README.md b/integration_tests/README.md new file mode 100644 index 00000000..3ef78b64 --- /dev/null +++ b/integration_tests/README.md @@ -0,0 +1,7 @@ +## Integration tests + +The files in `public_data` are available in two public storage buckets: +- `s3://dbt-external-tables-testing` +- `gs://dbt-external-tables-testing/` + +These integration tests confirm that, when staged as external tables, using different databases / file formats / partitioning schemes, the final combined output is equivalent to `seeds/people.csv`. diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index babc3cac..42cd38c0 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,16 +6,16 @@ profile: 'integration_tests' config-version: 2 -source-paths: ["models"] +model-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["seeds"] macro-paths: ["macros"] target-path: "target" clean-targets: - - "target" - - "dbt_modules" + - "target" + - "dbt_packages" dispatch: - macro_namespace: dbt_external_tables diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml index 7468ac55..2328e8c9 100644 --- a/integration_tests/packages.yml +++ b/integration_tests/packages.yml @@ -1,3 +1,5 @@ packages: - - local: ../ + - local: ../ + - package: dbt-labs/dbt_utils + version: 0.8.0 diff --git a/integration_tests/data/people.csv b/integration_tests/seeds/people.csv similarity index 100% rename from integration_tests/data/people.csv rename to integration_tests/seeds/people.csv diff --git a/macros/common/stage_external_sources.sql b/macros/common/stage_external_sources.sql index 13b807b6..7fad72d5 100644 --- a/macros/common/stage_external_sources.sql +++ b/macros/common/stage_external_sources.sql @@ -34,26 +34,26 @@ {% endfor %} {% if sources_to_stage|length == 0 %} - {% do dbt_utils.log_info('No external sources selected') %} + {% do log('No external sources selected', info = true) %} {% endif %} {% for node in sources_to_stage %} {% set loop_label = loop.index ~ ' of ' ~ loop.length %} - {% do dbt_utils.log_info(loop_label ~ ' START external source ' ~ node.schema ~ '.' ~ node.identifier) -%} + {% do log(loop_label ~ ' START external source ' ~ node.schema ~ '.' ~ node.identifier, info = true) -%} {% set run_queue = dbt_external_tables.get_external_build_plan(node) %} - {% do dbt_utils.log_info(loop_label ~ ' SKIP') if run_queue == [] %} - {% set width = 80 %} {# hard code this for now, use PRINTER_WIDTH flag in v1.0+ #} + {% do log(loop_label ~ ' SKIP', info = true) if run_queue == [] %} + {% set width = flags.PRINTER_WIDTH %} {% for q in run_queue %} {% set q_msg = q|replace('\n','')|replace('begin;','')|trim %} {% set q_log = q_msg[:width] ~ '... ' if q_msg|length > width else q_msg %} - {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log) %} + {% do log(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log, info = true) %} {% set exit_txn = dbt_external_tables.exit_transaction() %} {% call statement('runner', fetch_result = True, auto_begin = False) %} @@ -62,7 +62,7 @@ {% set runner = load_result('runner') %} {% set log_msg = runner['response'] if 'response' in runner.keys() else runner['status'] %} - {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ log_msg) %} + {% do log(loop_label ~ ' (' ~ loop.index ~ ') ' ~ log_msg, info = true) %} {% endfor %} diff --git a/packages.yml b/packages.yml deleted file mode 100644 index 5302a1f2..00000000 --- a/packages.yml +++ /dev/null @@ -1,3 +0,0 @@ -packages: - - package: dbt-labs/dbt_utils - version: [">=0.7.0", "<0.8.0"]