diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 24e904ac0..bd9430cbe 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0a1 +current_version = 1.9.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md new file mode 100644 index 000000000..7d0dd2c8f --- /dev/null +++ b/.changes/1.9.0-b1.md @@ -0,0 +1,44 @@ +## dbt-bigquery 1.9.0-b1 - October 02, 2024 + +### Features + +- Add configuration options `enable_list_inference` and `intermediate_format` for python models ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Add tests for cross-database `cast` macro ([#1214](https://github.com/dbt-labs/dbt-bigquery/issues/1214)) +- Cross-database `date` macro ([#1221](https://github.com/dbt-labs/dbt-bigquery/issues/1221)) +- Add support for base 64 encoded json keyfile credentials ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- Add support for cancelling queries on keyboard interrupt ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- Add Microbatch Strategy to dbt-spark ([#1354](https://github.com/dbt-labs/dbt-bigquery/issues/1354)) + +### Fixes + +- Drop intermediate objects created in BigQuery for incremental models ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) +- Fix null column index issue during `dbt docs generate` for external tables ([#1079](https://github.com/dbt-labs/dbt-bigquery/issues/1079)) +- make seed delimiter configurable via `field_delimeter` in model config ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- Default `enableListInference` to `True` for python models to support nested lists ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Catch additional database error exception, NotFound, as a DbtDatabaseError instead of defaulting to a DbtRuntimeError ([#1360](https://github.com/dbt-labs/dbt-bigquery/issues/1360)) + +### Under the Hood + +- Lazy load `agate` ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- Simplify linting environment and dev dependencies ([#1291](https://github.com/dbt-labs/dbt-bigquery/issues/1291)) + +### Dependencies + +- Update pre-commit requirement from ~=3.5 to ~=3.7 ([#1052](https://github.com/dbt-labs/dbt-bigquery/pull/1052)) +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#1062](https://github.com/dbt-labs/dbt-bigquery/pull/1062)) +- Bump mypy from 1.7.1 to 1.8.0 ([#1064](https://github.com/dbt-labs/dbt-bigquery/pull/1064)) +- Update flake8 requirement from ~=6.1 to ~=7.0 ([#1069](https://github.com/dbt-labs/dbt-bigquery/pull/1069)) +- Bump actions/download-artifact from 3 to 4 ([#1209](https://github.com/dbt-labs/dbt-bigquery/pull/1209)) +- Bump actions/upload-artifact from 3 to 4 ([#1210](https://github.com/dbt-labs/dbt-bigquery/pull/1210)) +- Bump ubuntu from 22.04 to 24.04 in /docker ([#1247](https://github.com/dbt-labs/dbt-bigquery/pull/1247)) +- Update pre-commit-hooks requirement from ~=4.5 to ~=4.6 ([#1281](https://github.com/dbt-labs/dbt-bigquery/pull/1281)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1282](https://github.com/dbt-labs/dbt-bigquery/pull/1282)) +- Update flaky requirement from ~=3.7 to ~=3.8 ([#1283](https://github.com/dbt-labs/dbt-bigquery/pull/1283)) +- Update twine requirement from ~=4.0 to ~=5.1 ([#1293](https://github.com/dbt-labs/dbt-bigquery/pull/1293)) + +### Contributors +- [@d-cole](https://github.com/d-cole) ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- [@dwreeves](https://github.com/dwreeves) ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- [@robeleb1](https://github.com/robeleb1) ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- [@salimmoulouel](https://github.com/salimmoulouel) ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- [@vinit2107](https://github.com/vinit2107) ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) diff --git a/.changes/unreleased/Dependencies-20231211-001048.yaml b/.changes/1.9.0/Dependencies-20231211-001048.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231211-001048.yaml rename to .changes/1.9.0/Dependencies-20231211-001048.yaml diff --git a/.changes/unreleased/Dependencies-20231220-002130.yaml b/.changes/1.9.0/Dependencies-20231220-002130.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231220-002130.yaml rename to .changes/1.9.0/Dependencies-20231220-002130.yaml diff --git a/.changes/unreleased/Dependencies-20231222-002351.yaml b/.changes/1.9.0/Dependencies-20231222-002351.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231222-002351.yaml rename to .changes/1.9.0/Dependencies-20231222-002351.yaml diff --git a/.changes/unreleased/Dependencies-20240105-004800.yaml b/.changes/1.9.0/Dependencies-20240105-004800.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240105-004800.yaml rename to .changes/1.9.0/Dependencies-20240105-004800.yaml diff --git a/.changes/unreleased/Dependencies-20240429-005158.yaml b/.changes/1.9.0/Dependencies-20240429-005158.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240429-005158.yaml rename to .changes/1.9.0/Dependencies-20240429-005158.yaml diff --git a/.changes/unreleased/Dependencies-20240429-005159.yaml b/.changes/1.9.0/Dependencies-20240429-005159.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240429-005159.yaml rename to .changes/1.9.0/Dependencies-20240429-005159.yaml diff --git a/.changes/unreleased/Dependencies-20240520-230208.yaml b/.changes/1.9.0/Dependencies-20240520-230208.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240520-230208.yaml rename to .changes/1.9.0/Dependencies-20240520-230208.yaml diff --git a/.changes/unreleased/Dependencies-20240718-005755.yaml b/.changes/1.9.0/Dependencies-20240718-005755.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-005755.yaml rename to .changes/1.9.0/Dependencies-20240718-005755.yaml diff --git a/.changes/unreleased/Dependencies-20240718-005756.yaml b/.changes/1.9.0/Dependencies-20240718-005756.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-005756.yaml rename to .changes/1.9.0/Dependencies-20240718-005756.yaml diff --git a/.changes/unreleased/Dependencies-20240718-005757.yaml b/.changes/1.9.0/Dependencies-20240718-005757.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-005757.yaml rename to .changes/1.9.0/Dependencies-20240718-005757.yaml diff --git a/.changes/unreleased/Dependencies-20240719-003740.yaml b/.changes/1.9.0/Dependencies-20240719-003740.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240719-003740.yaml rename to .changes/1.9.0/Dependencies-20240719-003740.yaml diff --git a/.changes/unreleased/Features-20240426-105319.yaml b/.changes/1.9.0/Features-20240426-105319.yaml similarity index 100% rename from .changes/unreleased/Features-20240426-105319.yaml rename to .changes/1.9.0/Features-20240426-105319.yaml diff --git a/.changes/unreleased/Features-20240430-185650.yaml b/.changes/1.9.0/Features-20240430-185650.yaml similarity index 100% rename from .changes/unreleased/Features-20240430-185650.yaml rename to .changes/1.9.0/Features-20240430-185650.yaml diff --git a/.changes/unreleased/Features-20240501-151902.yaml b/.changes/1.9.0/Features-20240501-151902.yaml similarity index 100% rename from .changes/unreleased/Features-20240501-151902.yaml rename to .changes/1.9.0/Features-20240501-151902.yaml diff --git a/.changes/unreleased/Features-20240516-125735.yaml b/.changes/1.9.0/Features-20240516-125735.yaml similarity index 100% rename from .changes/unreleased/Features-20240516-125735.yaml rename to .changes/1.9.0/Features-20240516-125735.yaml diff --git a/.changes/unreleased/Features-20240730-135911.yaml b/.changes/1.9.0/Features-20240730-135911.yaml similarity index 100% rename from .changes/unreleased/Features-20240730-135911.yaml rename to .changes/1.9.0/Features-20240730-135911.yaml diff --git a/.changes/unreleased/Features-20240925-232238.yaml b/.changes/1.9.0/Features-20240925-232238.yaml similarity index 100% rename from .changes/unreleased/Features-20240925-232238.yaml rename to .changes/1.9.0/Features-20240925-232238.yaml diff --git a/.changes/unreleased/Fixes-20240120-180818.yaml b/.changes/1.9.0/Fixes-20240120-180818.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240120-180818.yaml rename to .changes/1.9.0/Fixes-20240120-180818.yaml diff --git a/.changes/unreleased/Fixes-20240201-145323.yaml b/.changes/1.9.0/Fixes-20240201-145323.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240201-145323.yaml rename to .changes/1.9.0/Fixes-20240201-145323.yaml diff --git a/.changes/unreleased/Fixes-20240226-233024.yaml b/.changes/1.9.0/Fixes-20240226-233024.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240226-233024.yaml rename to .changes/1.9.0/Fixes-20240226-233024.yaml diff --git a/.changes/unreleased/Fixes-20240426-105224.yaml b/.changes/1.9.0/Fixes-20240426-105224.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240426-105224.yaml rename to .changes/1.9.0/Fixes-20240426-105224.yaml diff --git a/.changes/1.9.0/Fixes-20241001-193207.yaml b/.changes/1.9.0/Fixes-20241001-193207.yaml new file mode 100644 index 000000000..584445a5b --- /dev/null +++ b/.changes/1.9.0/Fixes-20241001-193207.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Catch additional database error exception, NotFound, as a DbtDatabaseError instead + of defaulting to a DbtRuntimeError +time: 2024-10-01T19:32:07.304353-04:00 +custom: + Author: mikealfare + Issue: "1360" diff --git a/.changes/unreleased/Under the Hood-20240331-101418.yaml b/.changes/1.9.0/Under the Hood-20240331-101418.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240331-101418.yaml rename to .changes/1.9.0/Under the Hood-20240331-101418.yaml diff --git a/.changes/unreleased/Under the Hood-20240718-193206.yaml b/.changes/1.9.0/Under the Hood-20240718-193206.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240718-193206.yaml rename to .changes/1.9.0/Under the Hood-20240718-193206.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-185117.yaml b/.changes/unreleased/Breaking Changes-20241016-185117.yaml new file mode 100644 index 000000000..55bb37461 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-185117.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:51:17.581547-04:00 +custom: + Author: mikealfare + Issue: "1373" diff --git a/.changes/unreleased/Features-20240505-011838.yaml b/.changes/unreleased/Features-20240505-011838.yaml new file mode 100644 index 000000000..66411853f --- /dev/null +++ b/.changes/unreleased/Features-20240505-011838.yaml @@ -0,0 +1,6 @@ +kind: Features +body: add is_retryable test case when raise ServiceUnavailable +time: 2024-05-05T01:18:38.737882+09:00 +custom: + Author: jx2lee + Issue: "682" diff --git a/.changes/unreleased/Features-20240911-234859.yaml b/.changes/unreleased/Features-20240911-234859.yaml new file mode 100644 index 000000000..5351c3315 --- /dev/null +++ b/.changes/unreleased/Features-20240911-234859.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Adds the ability to set optional `quota_project` in profile +time: 2024-09-11T23:48:59.767649+01:00 +custom: + Author: jcarpenter12 + Issue: 1343 1344 diff --git a/.changes/unreleased/Fixes-20241028-172719.yaml b/.changes/unreleased/Fixes-20241028-172719.yaml new file mode 100644 index 000000000..87ee2c25d --- /dev/null +++ b/.changes/unreleased/Fixes-20241028-172719.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: use "direct" write for non-partitioned python model materializations +time: 2024-10-28T17:27:19.306348-07:00 +custom: + Author: colin-rogers-dbt + Issue: "1318" diff --git a/.changes/unreleased/Under the Hood-20240910-212052.yaml b/.changes/unreleased/Under the Hood-20240910-212052.yaml new file mode 100644 index 000000000..3e4885dcd --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240910-212052.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Isolating distribution testing +time: 2024-09-10T21:20:52.574204-04:00 +custom: + Author: leahwicz + Issue: "1290" diff --git a/.github/scripts/integration-test-matrix.js b/.github/scripts/integration-test-matrix.js index 8e4b351fc..49db45575 100644 --- a/.github/scripts/integration-test-matrix.js +++ b/.github/scripts/integration-test-matrix.js @@ -1,6 +1,6 @@ module.exports = ({ context }) => { - const defaultPythonVersion = "3.8"; - const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"]; + const defaultPythonVersion = "3.9"; + const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"]; const supportedAdapters = ["bigquery"]; // if PR, generate matrix based on files changed and PR labels diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 43ac4ecb3..a9179f9ce 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,6 +33,7 @@ on: # all PRs, important to note that `pull_request_target` workflows # will run in the context of the target branch of a PR pull_request_target: + types: [opened, reopened, synchronize, labeled] # manual trigger workflow_dispatch: inputs: @@ -280,10 +281,10 @@ jobs: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.9" - name: Install python dependencies run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d04b4307f..6bfed5df6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -70,7 +70,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] env: TOXENV: "unit" @@ -127,7 +127,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -163,7 +163,7 @@ jobs: overwrite: true test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} + name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} if: needs.build.outputs.is_alpha == 0 @@ -175,18 +175,21 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12, windows-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] + dist-type: ["whl", "gz"] steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install python dependencies run: | python -m pip install --user --upgrade pip - python -m pip install --upgrade wheel setuptools twine check-wheel-contents + python -m pip install --upgrade wheel python -m pip --version + - uses: actions/download-artifact@v4 with: name: dist @@ -195,15 +198,10 @@ jobs: - name: Show distributions run: ls -lh dist/ - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check wheel distributions + - name: Install ${{ matrix.dist-type }} distributions run: | - python -c "import dbt.adapters.bigquery" - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check source distributions + find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check ${{ matrix.dist-type }} distributions run: | python -c "import dbt.adapters.bigquery" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d0b98d45..16760bf07 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a408c580..b9bda350a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,54 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-bigquery/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-bigquery 1.9.0-b1 - October 02, 2024 + +### Features + +- Add configuration options `enable_list_inference` and `intermediate_format` for python models ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Add tests for cross-database `cast` macro ([#1214](https://github.com/dbt-labs/dbt-bigquery/issues/1214)) +- Cross-database `date` macro ([#1221](https://github.com/dbt-labs/dbt-bigquery/issues/1221)) +- Add support for base 64 encoded json keyfile credentials ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- Add support for cancelling queries on keyboard interrupt ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- Add Microbatch Strategy to dbt-spark ([#1354](https://github.com/dbt-labs/dbt-bigquery/issues/1354)) + +### Fixes + +- Drop intermediate objects created in BigQuery for incremental models ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) +- Fix null column index issue during `dbt docs generate` for external tables ([#1079](https://github.com/dbt-labs/dbt-bigquery/issues/1079)) +- make seed delimiter configurable via `field_delimeter` in model config ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- Default `enableListInference` to `True` for python models to support nested lists ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114)) +- Catch additional database error exception, NotFound, as a DbtDatabaseError instead of defaulting to a DbtRuntimeError ([#1360](https://github.com/dbt-labs/dbt-bigquery/issues/1360)) + +### Under the Hood + +- Lazy load `agate` ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- Simplify linting environment and dev dependencies ([#1291](https://github.com/dbt-labs/dbt-bigquery/issues/1291)) + +### Dependencies + +- Update pre-commit requirement from ~=3.5 to ~=3.7 ([#1052](https://github.com/dbt-labs/dbt-bigquery/pull/1052)) +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#1062](https://github.com/dbt-labs/dbt-bigquery/pull/1062)) +- Bump mypy from 1.7.1 to 1.8.0 ([#1064](https://github.com/dbt-labs/dbt-bigquery/pull/1064)) +- Update flake8 requirement from ~=6.1 to ~=7.0 ([#1069](https://github.com/dbt-labs/dbt-bigquery/pull/1069)) +- Bump actions/download-artifact from 3 to 4 ([#1209](https://github.com/dbt-labs/dbt-bigquery/pull/1209)) +- Bump actions/upload-artifact from 3 to 4 ([#1210](https://github.com/dbt-labs/dbt-bigquery/pull/1210)) +- Bump ubuntu from 22.04 to 24.04 in /docker ([#1247](https://github.com/dbt-labs/dbt-bigquery/pull/1247)) +- Update pre-commit-hooks requirement from ~=4.5 to ~=4.6 ([#1281](https://github.com/dbt-labs/dbt-bigquery/pull/1281)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1282](https://github.com/dbt-labs/dbt-bigquery/pull/1282)) +- Update flaky requirement from ~=3.7 to ~=3.8 ([#1283](https://github.com/dbt-labs/dbt-bigquery/pull/1283)) +- Update twine requirement from ~=4.0 to ~=5.1 ([#1293](https://github.com/dbt-labs/dbt-bigquery/pull/1293)) + +### Contributors +- [@d-cole](https://github.com/d-cole) ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917)) +- [@dwreeves](https://github.com/dwreeves) ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162)) +- [@robeleb1](https://github.com/robeleb1) ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923)) +- [@salimmoulouel](https://github.com/salimmoulouel) ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119)) +- [@vinit2107](https://github.com/vinit2107) ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: -- [1.8](https://github.com/dbt-labs/dbt-bigquery/blob/1.8.latest/CHANGELOG.md) -- [1.7](https://github.com/dbt-labs/dbt-bigquery/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-bigquery/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-bigquery/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-bigquery/blob/1.4.latest/CHANGELOG.md) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1af648741..f915af713 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ To confirm you have the correct version of `dbt-core` installed please run `dbt ### Initial Setup -`dbt-bigquery` contains [unit](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/unit) and [integration](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/integration) tests. Integration tests require testing against an actual BigQuery warehouse. We have CI set up to test against a BigQuery warehouse. In order to run integration tests locally, you will need a `test.env` file in the root of the repository that contains credentials for BigQuery. +`dbt-bigquery` contains [unit](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/functional) tests. functional tests require testing against an actual BigQuery warehouse. We have CI set up to test against a BigQuery warehouse. In order to run functional tests locally, you will need a `test.env` file in the root of the repository that contains credentials for BigQuery. Note: This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing. To create your `test.env` file, copy the provided example file, then supply your relevant credentials. @@ -67,7 +67,7 @@ $EDITOR test.env There are a few methods for running tests locally. #### `tox` -`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.8, Python 3.9, Python 3.10, and Python 3.11 in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py38`. The configuration of these tests are located in `tox.ini`. +`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.9, Python 3.10, and Python 3.11 in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py39`. The configuration of these tests are located in `tox.ini`. #### `pytest` Finally, you can also run a specific test or group of tests using `pytest` directly. With a Python virtualenv active and dev dependencies installed you can do things like: @@ -104,6 +104,6 @@ You don't need to worry about which `dbt-bigquery` version your change will go i dbt Labs provides a CI environment to test changes to the `dbt-bigquery` adapter and periodic checks against the development version of `dbt-core` through Github Actions. -A `dbt-bigquery` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. +A `dbt-bigquery` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or functional test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. Once all tests are passing, you have updated the changelog to reflect and tag your issue/pr for reference with a small description of the change, and your PR has been approved, a `dbt-bigquery` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: diff --git a/dbt/adapters/bigquery/__version__.py b/dbt/adapters/bigquery/__version__.py index 6698ed64c..a4077fff2 100644 --- a/dbt/adapters/bigquery/__version__.py +++ b/dbt/adapters/bigquery/__version__.py @@ -1 +1 @@ -version = "1.9.0a1" +version = "1.9.0b1" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index cdd9d17dc..58b3dbe41 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -17,7 +17,7 @@ import google.auth.exceptions import google.cloud.bigquery import google.cloud.exceptions -from google.api_core import retry, client_info +from google.api_core import retry, client_info, client_options from google.auth import impersonated_credentials from google.oauth2 import ( credentials as GoogleCredentials, @@ -125,6 +125,7 @@ class BigQueryCredentials(Credentials): database: Optional[str] = None schema: Optional[str] = None execution_project: Optional[str] = None + quota_project: Optional[str] = None location: Optional[str] = None priority: Optional[Priority] = None maximum_bytes_billed: Optional[int] = None @@ -268,6 +269,10 @@ def exception_handler(self, sql): message = "Access denied while running query" self.handle_error(e, message) + except google.cloud.exceptions.NotFound as e: + message = "Not found while running query" + self.handle_error(e, message) + except google.auth.exceptions.RefreshError as e: message = ( "Unable to generate access token, if you're using " @@ -404,14 +409,17 @@ def get_credentials(cls, profile_credentials): def get_bigquery_client(cls, profile_credentials): creds = cls.get_credentials(profile_credentials) execution_project = profile_credentials.execution_project + quota_project = profile_credentials.quota_project location = getattr(profile_credentials, "location", None) info = client_info.ClientInfo(user_agent=f"dbt-bigquery-{dbt_version.version}") + options = client_options.ClientOptions(quota_project_id=quota_project) return google.cloud.bigquery.Client( execution_project, creds, location=location, client_info=info, + client_options=options, ) @classmethod diff --git a/dbt/include/bigquery/macros/materializations/table.sql b/dbt/include/bigquery/macros/materializations/table.sql index e3c5b3598..41bb69770 100644 --- a/dbt/include/bigquery/macros/materializations/table.sql +++ b/dbt/include/bigquery/macros/materializations/table.sql @@ -113,10 +113,19 @@ else: msg = f"{type(df)} is not a supported type for dbt Python materialization" raise Exception(msg) +# For writeMethod we need to use "indirect" if materializing a partitioned table +# otherwise we can use "direct". Note that indirect will fail if the GCS bucket has a retention policy set on it. +{%- if partition_config %} + {%- set write_method = 'indirect' -%} +{%- else %} + {% set write_method = 'direct' -%} +{%- endif %} + df.write \ .mode("overwrite") \ .format("bigquery") \ - .option("writeMethod", "indirect").option("writeDisposition", 'WRITE_TRUNCATE') \ + .option("writeMethod", "{{ write_method }}") \ + .option("writeDisposition", 'WRITE_TRUNCATE') \ {%- if partition_config is not none %} {%- if partition_config.data_type | lower in ('date','timestamp','datetime') %} .option("partitionField", "{{- partition_config.field -}}") \ diff --git a/dev-requirements.txt b/dev-requirements.txt index 34169172a..2c0134110 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,8 +6,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core # dev ddtrace==2.3.0 -pre-commit~=3.7.0;python_version>="3.9" -pre-commit~=3.5.0;python_version<"3.9" +pre-commit~=3.7.0 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 diff --git a/docker/Dockerfile b/docker/Dockerfile index 3b9431fd1..bda507dc5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # this image gets published to GHCR for production use ARG py_version=3.11.2 -FROM python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -25,7 +25,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir -FROM base as dbt-bigquery +FROM base AS dbt-bigquery ARG commit_ref=main diff --git a/docker/dev.Dockerfile b/docker/dev.Dockerfile index 2afad0a95..f122f5343 100644 --- a/docker/dev.Dockerfile +++ b/docker/dev.Dockerfile @@ -1,43 +1,43 @@ # this image does not get published, it is intended for local development only, see `Makefile` for usage -FROM ubuntu:24.04 as base +FROM ubuntu:24.04 AS base # prevent python installation from asking for time zone region ARG DEBIAN_FRONTEND=noninteractive # add python repository RUN apt-get update \ - && apt-get install -y software-properties-common=0.99.22.9 \ - && add-apt-repository -y ppa:deadsnakes/ppa \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y software-properties-common=0.99.48 \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # install python RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - build-essential=12.9ubuntu3 \ - git-all=1:2.34.1-1ubuntu1.10 \ - python3.8=3.8.19-1+jammy1 \ - python3.8-dev=3.8.19-1+jammy1 \ - python3.8-distutils=3.8.19-1+jammy1 \ - python3.8-venv=3.8.19-1+jammy1 \ - python3-pip=22.0.2+dfsg-1ubuntu0.4 \ - python3-wheel=0.37.1-2ubuntu0.22.04.1 \ - && apt-get clean \ - && rm -rf \ - /var/lib/apt/lists/* \ - /tmp/* \ - /var/tmp/* + && apt-get install -y --no-install-recommends \ + build-essential=12.10ubuntu1 \ + git-all=1:2.43.0-1ubuntu7.1 \ + python3.9=3.9.20-1+noble1 \ + python3.9-dev=3.9.20-1+noble1 \ + python3.9-distutils=3.9.20-1+noble1 \ + python3.9-venv=3.9.20-1+noble1 \ + python3-pip=24.0+dfsg-1ubuntu1 \ + python3-wheel=0.42.0-2 \ + && apt-get clean \ + && rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* # update the default system interpreter to the newly installed version -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 -FROM base as dbt-bigquery-dev +FROM base AS dbt-bigquery-dev -HEALTHCHECK CMD python3 --version || exit 1 +HEALTHCHECK CMD python --version || exit 1 # send stdout/stderr to terminal ENV PYTHONUNBUFFERED=1 diff --git a/setup.py b/setup.py index ab89a3c39..79f6025ea 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ import sys # require a supported version of Python -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) try: @@ -69,11 +69,10 @@ def _dbt_bigquery_version() -> str: "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", ) diff --git a/tests/functional/test_quota_project.py b/tests/functional/test_quota_project.py new file mode 100644 index 000000000..0b4bb90c4 --- /dev/null +++ b/tests/functional/test_quota_project.py @@ -0,0 +1,27 @@ +import os + +import pytest + +from dbt.tests.util import run_dbt + +_QUOTA_PROJECT = os.getenv("BIGQUERY_TEST_ALT_DATABASE") + + +class TestNoQuotaProject: + def test_no_quota_project(self, project): + results = run_dbt() + for result in results: + assert None == result.adapter_response["quota_project"] + + +class TestQuotaProjectOption: + @pytest.fixture(scope="class") + def profiles_config_update(self, dbt_profile_target): + outputs = {"default": dbt_profile_target} + outputs["default"]["quota_project"] = _QUOTA_PROJECT + yield + + def test_quota_project_option(self, project): + results = run_dbt() + for result in results: + assert _QUOTA_PROJECT == result.adapter_response["quota_project"] diff --git a/tests/unit/test_bigquery_adapter.py b/tests/unit/test_bigquery_adapter.py index a922525fd..da499d266 100644 --- a/tests/unit/test_bigquery_adapter.py +++ b/tests/unit/test_bigquery_adapter.py @@ -386,15 +386,17 @@ def test_cancel_open_connections_single(self): adapter.connections.thread_connections.update({key: master, 1: model}) self.assertEqual(len(list(adapter.cancel_open_connections())), 1) + @patch("dbt.adapters.bigquery.impl.google.api_core.client_options.ClientOptions") @patch("dbt.adapters.bigquery.impl.google.auth.default") @patch("dbt.adapters.bigquery.impl.google.cloud.bigquery") - def test_location_user_agent(self, mock_bq, mock_auth_default): + def test_location_user_agent(self, mock_bq, mock_auth_default, MockClientOptions): creds = MagicMock() mock_auth_default.return_value = (creds, MagicMock()) adapter = self.get_adapter("loc") connection = adapter.acquire_connection("dummy") mock_client = mock_bq.Client + mock_client_options = MockClientOptions.return_value mock_client.assert_not_called() connection.handle @@ -403,6 +405,7 @@ def test_location_user_agent(self, mock_bq, mock_auth_default): creds, location="Luna Station", client_info=HasUserAgent(), + client_options=mock_client_options, ) diff --git a/tests/unit/test_bigquery_connection_manager.py b/tests/unit/test_bigquery_connection_manager.py index d09cb1635..1c14100f6 100644 --- a/tests/unit/test_bigquery_connection_manager.py +++ b/tests/unit/test_bigquery_connection_manager.py @@ -84,12 +84,14 @@ def test_is_retryable(self): rate_limit_error = exceptions.Forbidden( "code broke", errors=[{"reason": "rateLimitExceeded"}] ) + service_unavailable_error = exceptions.ServiceUnavailable("service is unavailable") self.assertTrue(_is_retryable(internal_server_error)) self.assertTrue(_is_retryable(bad_request_error)) self.assertTrue(_is_retryable(connection_error)) self.assertFalse(_is_retryable(client_error)) self.assertTrue(_is_retryable(rate_limit_error)) + self.assertTrue(_is_retryable(service_unavailable_error)) def test_drop_dataset(self): mock_table = Mock() diff --git a/tox.ini b/tox.ini index b388dc5b3..240d85e34 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] skipsdist = True -envlist = py38,py39,py310,py311,py312 +envlist = py39,py310,py311,py312 -[testenv:{unit,py38,py39,py310,py311,py312,py}] +[testenv:{unit,py39,py310,py311,py312,py}] description = unit testing skip_install = true passenv = @@ -13,7 +13,7 @@ deps = -rdev-requirements.txt -e. -[testenv:{integration,py38,py39,py310,py311,py312,py}-{bigquery}] +[testenv:{integration,py39,py310,py311,py312,py}-{bigquery}] description = adapter plugin integration testing skip_install = true passenv = @@ -33,7 +33,7 @@ deps = -rdev-requirements.txt . -[testenv:{python-tests,py38,py39,py310,py311,py312,py}] +[testenv:{python-tests,py39,py310,py311,py312,py}] description = python integration testing skip_install = true passenv =