Merge branch 'main' into feature/add-quota-project-option

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.0a1
+current_version = 1.9.0b1
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.9.0-b1.md

@@ -0,0 +1,44 @@
+## dbt-bigquery 1.9.0-b1 - October 02, 2024
+
+### Features
+
+- Add configuration options `enable_list_inference` and `intermediate_format` for python models ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114))
+- Add tests for cross-database `cast` macro ([#1214](https://github.com/dbt-labs/dbt-bigquery/issues/1214))
+- Cross-database `date` macro ([#1221](https://github.com/dbt-labs/dbt-bigquery/issues/1221))
+- Add support for base 64 encoded json keyfile credentials ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923))
+- Add support for cancelling queries on keyboard interrupt ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917))
+- Add Microbatch Strategy to dbt-spark ([#1354](https://github.com/dbt-labs/dbt-bigquery/issues/1354))
+
+### Fixes
+
+- Drop intermediate objects created in BigQuery for incremental models ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036))
+- Fix null column index issue during `dbt docs generate` for external tables ([#1079](https://github.com/dbt-labs/dbt-bigquery/issues/1079))
+- make seed delimiter configurable via `field_delimeter` in model config ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119))
+- Default `enableListInference` to `True` for python models to support nested lists ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114))
+- Catch additional database error exception, NotFound, as a DbtDatabaseError instead of defaulting to a DbtRuntimeError ([#1360](https://github.com/dbt-labs/dbt-bigquery/issues/1360))
+
+### Under the Hood
+
+- Lazy load `agate` ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162))
+- Simplify linting environment and dev dependencies ([#1291](https://github.com/dbt-labs/dbt-bigquery/issues/1291))
+
+### Dependencies
+
+- Update pre-commit requirement from ~=3.5 to ~=3.7 ([#1052](https://github.com/dbt-labs/dbt-bigquery/pull/1052))
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#1062](https://github.com/dbt-labs/dbt-bigquery/pull/1062))
+- Bump mypy from 1.7.1 to 1.8.0 ([#1064](https://github.com/dbt-labs/dbt-bigquery/pull/1064))
+- Update flake8 requirement from ~=6.1 to ~=7.0 ([#1069](https://github.com/dbt-labs/dbt-bigquery/pull/1069))
+- Bump actions/download-artifact from 3 to 4 ([#1209](https://github.com/dbt-labs/dbt-bigquery/pull/1209))
+- Bump actions/upload-artifact from 3 to 4 ([#1210](https://github.com/dbt-labs/dbt-bigquery/pull/1210))
+- Bump ubuntu from 22.04 to 24.04 in /docker ([#1247](https://github.com/dbt-labs/dbt-bigquery/pull/1247))
+- Update pre-commit-hooks requirement from ~=4.5 to ~=4.6 ([#1281](https://github.com/dbt-labs/dbt-bigquery/pull/1281))
+- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1282](https://github.com/dbt-labs/dbt-bigquery/pull/1282))
+- Update flaky requirement from ~=3.7 to ~=3.8 ([#1283](https://github.com/dbt-labs/dbt-bigquery/pull/1283))
+- Update twine requirement from ~=4.0 to ~=5.1 ([#1293](https://github.com/dbt-labs/dbt-bigquery/pull/1293))
+
+### Contributors
+- [@d-cole](https://github.com/d-cole) ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917))
+- [@dwreeves](https://github.com/dwreeves) ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162))
+- [@robeleb1](https://github.com/robeleb1) ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923))
+- [@salimmoulouel](https://github.com/salimmoulouel) ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119))
+- [@vinit2107](https://github.com/vinit2107) ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036))

.changes/1.9.0/Features-20240925-232238.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add Microbatch Strategy to dbt-spark
+time: 2024-09-25T23:22:38.216277+01:00
+custom:
+  Author: michelleark
+  Issue: "1354"

.changes/1.9.0/Fixes-20241001-193207.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Catch additional database error exception, NotFound, as a DbtDatabaseError instead
+  of defaulting to a DbtRuntimeError
+time: 2024-10-01T19:32:07.304353-04:00
+custom:
+  Author: mikealfare
+  Issue: "1360"

.changes/unreleased/Breaking Changes-20241016-185117.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Drop support for Python 3.8
+time: 2024-10-16T18:51:17.581547-04:00
+custom:
+  Author: mikealfare
+  Issue: "1373"

.changes/unreleased/Under the Hood-20240910-212052.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Isolating distribution testing
+time: 2024-09-10T21:20:52.574204-04:00
+custom:
+    Author: leahwicz
+    Issue: "1290"

.github/scripts/integration-test-matrix.js

@@ -1,6 +1,6 @@
 module.exports = ({ context }) => {
-  const defaultPythonVersion = "3.8";
-  const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"];
+  const defaultPythonVersion = "3.9";
+  const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"];
   const supportedAdapters = ["bigquery"];
 
   // if PR, generate matrix based on files changed and PR labels

.github/workflows/integration.yml

@@ -33,6 +33,7 @@ on:
   # all PRs, important to note that `pull_request_target` workflows
   # will run in the context of the target branch of a PR
   pull_request_target:
+    types: [opened, reopened, synchronize, labeled]
   # manual trigger
   workflow_dispatch:
     inputs:
@@ -128,6 +129,7 @@ jobs:
               - 'tests/**'
               - 'dev-requirements.txt'
               - '.github/**'
+              - '*.py'
 
       - name: Generate integration test matrix
         id: generate-matrix
@@ -279,10 +281,10 @@ jobs:
           persist-credentials: false
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.9"
 
       - name: Install python dependencies
         run: |

.github/workflows/main.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -70,7 +70,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     env:
       TOXENV: "unit"
@@ -127,7 +127,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -163,7 +163,7 @@ jobs:
           overwrite: true
 
   test-build:
-    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }}
 
     if: needs.build.outputs.is_alpha == 0
 
@@ -175,18 +175,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+        dist-type: ["whl", "gz"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install python dependencies
         run: |
           python -m pip install --user --upgrade pip
-          python -m pip install --upgrade wheel setuptools twine check-wheel-contents
+          python -m pip install --upgrade wheel
           python -m pip --version
+
       - uses: actions/download-artifact@v4
         with:
           name: dist
@@ -195,15 +198,10 @@ jobs:
       - name: Show distributions
         run: ls -lh dist/
 
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check wheel distributions
+      - name: Install ${{ matrix.dist-type }} distributions
         run: |
-          python -c "import dbt.adapters.bigquery"
-      - name: Install source distributions
-        run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check source distributions
+          find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+
+      - name: Check ${{ matrix.dist-type }} distributions
         run: |
           python -c "import dbt.adapters.bigquery"

.pre-commit-config.yaml

@@ -24,7 +24,6 @@ repos:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311

CHANGELOG.md

@@ -5,10 +5,54 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-bigquery/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-bigquery 1.9.0-b1 - October 02, 2024
+
+### Features
+
+- Add configuration options `enable_list_inference` and `intermediate_format` for python models ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114))
+- Add tests for cross-database `cast` macro ([#1214](https://github.com/dbt-labs/dbt-bigquery/issues/1214))
+- Cross-database `date` macro ([#1221](https://github.com/dbt-labs/dbt-bigquery/issues/1221))
+- Add support for base 64 encoded json keyfile credentials ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923))
+- Add support for cancelling queries on keyboard interrupt ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917))
+- Add Microbatch Strategy to dbt-spark ([#1354](https://github.com/dbt-labs/dbt-bigquery/issues/1354))
+
+### Fixes
+
+- Drop intermediate objects created in BigQuery for incremental models ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036))
+- Fix null column index issue during `dbt docs generate` for external tables ([#1079](https://github.com/dbt-labs/dbt-bigquery/issues/1079))
+- make seed delimiter configurable via `field_delimeter` in model config ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119))
+- Default `enableListInference` to `True` for python models to support nested lists ([#1047](https://github.com/dbt-labs/dbt-bigquery/issues/1047), [#1114](https://github.com/dbt-labs/dbt-bigquery/issues/1114))
+- Catch additional database error exception, NotFound, as a DbtDatabaseError instead of defaulting to a DbtRuntimeError ([#1360](https://github.com/dbt-labs/dbt-bigquery/issues/1360))
+
+### Under the Hood
+
+- Lazy load `agate` ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162))
+- Simplify linting environment and dev dependencies ([#1291](https://github.com/dbt-labs/dbt-bigquery/issues/1291))
+
+### Dependencies
+
+- Update pre-commit requirement from ~=3.5 to ~=3.7 ([#1052](https://github.com/dbt-labs/dbt-bigquery/pull/1052))
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#1062](https://github.com/dbt-labs/dbt-bigquery/pull/1062))
+- Bump mypy from 1.7.1 to 1.8.0 ([#1064](https://github.com/dbt-labs/dbt-bigquery/pull/1064))
+- Update flake8 requirement from ~=6.1 to ~=7.0 ([#1069](https://github.com/dbt-labs/dbt-bigquery/pull/1069))
+- Bump actions/download-artifact from 3 to 4 ([#1209](https://github.com/dbt-labs/dbt-bigquery/pull/1209))
+- Bump actions/upload-artifact from 3 to 4 ([#1210](https://github.com/dbt-labs/dbt-bigquery/pull/1210))
+- Bump ubuntu from 22.04 to 24.04 in /docker ([#1247](https://github.com/dbt-labs/dbt-bigquery/pull/1247))
+- Update pre-commit-hooks requirement from ~=4.5 to ~=4.6 ([#1281](https://github.com/dbt-labs/dbt-bigquery/pull/1281))
+- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1282](https://github.com/dbt-labs/dbt-bigquery/pull/1282))
+- Update flaky requirement from ~=3.7 to ~=3.8 ([#1283](https://github.com/dbt-labs/dbt-bigquery/pull/1283))
+- Update twine requirement from ~=4.0 to ~=5.1 ([#1293](https://github.com/dbt-labs/dbt-bigquery/pull/1293))
+
+### Contributors
+- [@d-cole](https://github.com/d-cole) ([#917](https://github.com/dbt-labs/dbt-bigquery/issues/917))
+- [@dwreeves](https://github.com/dwreeves) ([#1162](https://github.com/dbt-labs/dbt-bigquery/issues/1162))
+- [@robeleb1](https://github.com/robeleb1) ([#923](https://github.com/dbt-labs/dbt-bigquery/issues/923))
+- [@salimmoulouel](https://github.com/salimmoulouel) ([#1119](https://github.com/dbt-labs/dbt-bigquery/issues/1119))
+- [@vinit2107](https://github.com/vinit2107) ([#1036](https://github.com/dbt-labs/dbt-bigquery/issues/1036))
+
+
 ## Previous Releases
 For information on prior major and minor releases, see their changelogs:
-- [1.8](https://github.com/dbt-labs/dbt-bigquery/blob/1.8.latest/CHANGELOG.md)
-- [1.7](https://github.com/dbt-labs/dbt-bigquery/blob/1.7.latest/CHANGELOG.md)
 - [1.6](https://github.com/dbt-labs/dbt-bigquery/blob/1.6.latest/CHANGELOG.md)
 - [1.5](https://github.com/dbt-labs/dbt-bigquery/blob/1.5.latest/CHANGELOG.md)
 - [1.4](https://github.com/dbt-labs/dbt-bigquery/blob/1.4.latest/CHANGELOG.md)

CONTRIBUTING.md

@@ -54,7 +54,7 @@ To confirm you have the correct version of `dbt-core` installed please run `dbt
 
 ### Initial Setup
 
-`dbt-bigquery` contains [unit](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/unit) and [integration](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/integration) tests. Integration tests require testing against an actual BigQuery warehouse. We have CI set up to test against a BigQuery warehouse. In order to run integration tests locally, you will need a `test.env` file in the root of the repository that contains credentials for BigQuery.
+`dbt-bigquery` contains [unit](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/unit) and [functional](https://github.com/dbt-labs/dbt-bigquery/tree/main/tests/functional) tests. functional tests require testing against an actual BigQuery warehouse. We have CI set up to test against a BigQuery warehouse. In order to run functional tests locally, you will need a `test.env` file in the root of the repository that contains credentials for BigQuery.
 
 Note: This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing. To create your `test.env` file, copy the provided example file, then supply your relevant credentials.
 
@@ -67,7 +67,7 @@ $EDITOR test.env
 There are a few methods for running tests locally.
 
 #### `tox`
-`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.8, Python 3.9, Python 3.10, and Python 3.11 in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py38`. The configuration of these tests are located in `tox.ini`.
+`tox` takes care of managing Python virtualenvs and installing dependencies in order to run tests. You can also run tests in parallel, for example you can run unit tests for Python 3.9, Python 3.10, and Python 3.11 in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py39`. The configuration of these tests are located in `tox.ini`.
 
 #### `pytest`
 Finally, you can also run a specific test or group of tests using `pytest` directly. With a Python virtualenv active and dev dependencies installed you can do things like:
@@ -104,6 +104,6 @@ You don't need to worry about which `dbt-bigquery` version your change will go i
 
 dbt Labs provides a CI environment to test changes to the `dbt-bigquery` adapter and periodic checks against the development version of `dbt-core` through Github Actions.
 
-A `dbt-bigquery` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
+A `dbt-bigquery` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or functional test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
 
 Once all tests are passing, you have updated the changelog to reflect and tag your issue/pr for reference with a small description of the change, and your PR has been approved, a `dbt-bigquery` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:

dbt/adapters/bigquery/__version__.py

@@ -1 +1 @@
-version = "1.9.0a1"
+version = "1.9.0b1"

dbt/adapters/bigquery/connections.py

@@ -269,6 +269,10 @@ def exception_handler(self, sql):
             message = "Access denied while running query"
             self.handle_error(e, message)
 
+        except google.cloud.exceptions.NotFound as e:
+            message = "Not found while running query"
+            self.handle_error(e, message)
+
         except google.auth.exceptions.RefreshError as e:
             message = (
                 "Unable to generate access token, if you're using "

dbt/include/bigquery/macros/materializations/incremental.sql

@@ -4,12 +4,16 @@
 
   {% set invalid_strategy_msg -%}
     Invalid incremental strategy provided: {{ strategy }}
-    Expected one of: 'merge', 'insert_overwrite'
+    Expected one of: 'merge', 'insert_overwrite', 'microbatch'
   {%- endset %}
-  {% if strategy not in ['merge', 'insert_overwrite'] %}
+  {% if strategy not in ['merge', 'insert_overwrite', 'microbatch'] %}
     {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}
   {% endif %}
 
+  {% if strategy == 'microbatch' %}
+    {% do bq_validate_microbatch_config(config) %}
+  {% endif %}
+
   {% do return(strategy) %}
 {% endmacro %}
 
@@ -48,8 +52,13 @@
         tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions
     ) %}
 
-  {% else %} {# strategy == 'merge' #}
+  {% elif strategy == 'microbatch' %}
 
+    {% set build_sql = bq_generate_microbatch_build_sql(
+        tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions
+    ) %}
+
+  {% else %} {# strategy == 'merge' #}
     {% set build_sql = bq_generate_incremental_merge_build_sql(
         tmp_relation, target_relation, sql, unique_key, partition_by, dest_columns, tmp_relation_exists, incremental_predicates
     ) %}

dbt/include/bigquery/macros/materializations/incremental_strategy/microbatch.sql

@@ -0,0 +1,28 @@
+{% macro bq_validate_microbatch_config(config) %}
+  {% if config.get("partition_by") is none %}
+    {% set missing_partition_msg -%}
+    The 'microbatch' strategy requires a `partition_by` config.
+    {%- endset %}
+    {% do exceptions.raise_compiler_error(missing_partition_msg) %}
+  {% endif %}
+
+  {% if config.get("partition_by").granularity != config.get('batch_size') %}
+    {% set invalid_partition_by_granularity_msg -%}
+    The 'microbatch' strategy requires a `partition_by` config with the same granularity as its configured `batch_size`.
+    Got:
+      `batch_size`: {{ config.get('batch_size') }}
+      `partition_by.granularity`: {{ config.get("partition_by").granularity }}
+    {%- endset %}
+    {% do exceptions.raise_compiler_error(invalid_partition_by_granularity_msg) %}
+  {% endif %}
+{% endmacro %}
+
+{% macro bq_generate_microbatch_build_sql(
+      tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions
+) %}
+    {% set build_sql = bq_insert_overwrite_sql(
+        tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists, copy_partitions
+    ) %}
+
+    {{ return(build_sql) }}
+{% endmacro %}

dev-requirements.txt

@@ -6,8 +6,7 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
 
 # dev
 ddtrace==2.3.0
-pre-commit~=3.7.0;python_version>="3.9"
-pre-commit~=3.5.0;python_version<"3.9"
+pre-commit~=3.7.0
 pytest~=7.4
 pytest-csv~=3.0
 pytest-dotenv~=0.5.2