From ddd4741b4061b70fa74f9cb6c9386fd5c7328366 Mon Sep 17 00:00:00 2001 From: matt-winkler <75497565+matt-winkler@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:35:27 -0600 Subject: [PATCH] Patch for `dbt show` for a model with a `struct` column containing a `json` datatype (#974) * patch for --show with json * update matts handle in changelog * Update dbt/adapters/bigquery/connections.py Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> * add json struct functional test * remove old change log * add comment to test --------- Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> Co-authored-by: Colin (cherry picked from commit 6a3f45897f4e7ccd41de584d64605e7342dc75e0) --- .../unreleased/Fixes-20231023-082312.yaml | 6 +++ dbt/adapters/bigquery/connections.py | 12 +++++- .../adapter/dbt_show/test_dbt_show.py | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/Fixes-20231023-082312.yaml diff --git a/.changes/unreleased/Fixes-20231023-082312.yaml b/.changes/unreleased/Fixes-20231023-082312.yaml new file mode 100644 index 000000000..368c58e95 --- /dev/null +++ b/.changes/unreleased/Fixes-20231023-082312.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Patch for json inline --show +time: 2023-10-23T08:23:12.245223-06:00 +custom: + Author: matt-winkler + Issue: "972" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 6ebd0477d..f38a8c58d 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -15,7 +15,7 @@ import google.auth import google.auth.exceptions -import google.cloud.bigquery +import google.cloud.bigquery as bigquery import google.cloud.exceptions from google.api_core import retry, client_info from google.auth import impersonated_credentials @@ -63,6 +63,16 @@ ) +# Override broken json deserializer for dbt show --inline +# can remove once this is fixed: https://github.com/googleapis/python-bigquery/issues/1500 +def _json_from_json(value, _): + """NOOP string -> string coercion""" + return json.loads(value) + + +bigquery._helpers._CELLDATA_FROM_JSON["JSON"] = _json_from_json + + @lru_cache() def get_bigquery_defaults(scopes=None) -> Tuple[Any, Optional[str]]: """ diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py index c60a26aec..203d7031b 100644 --- a/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -1,5 +1,29 @@ +import pytest from dbt.tests.adapter.dbt_show.test_dbt_show import BaseShowSqlHeader, BaseShowLimit +from dbt.tests.util import run_dbt + +model_with_json_struct = """ + select * + from ( + select + struct< + k array< + struct + > + >( + [ + struct( + 1 as c1, + to_json(struct(1 as a)) as c2 + ) + ] + ) + as v + ) as model_limit_subq + limit 5 + """ + class TestBigQueryShowLimit(BaseShowLimit): pass @@ -7,3 +31,16 @@ class TestBigQueryShowLimit(BaseShowLimit): class TestBigQueryShowSqlHeader(BaseShowSqlHeader): pass + + +# Added to check if dbt show works with JSON struct +# Addresses: https://github.com/dbt-labs/dbt-bigquery/issues/972 +class TestBigQueryShowSqlWorksWithJSONStruct: + @pytest.fixture(scope="class") + def models(self): + return { + "json_struct_model.sql": model_with_json_struct, + } + + def test_sql_header(self, project): + run_dbt(["show", "--select", "json_struct_model"])