diff --git a/dev-requirements.txt b/dev-requirements.txt index 44b1767d..98dbff78 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,3 +6,4 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugi git+https://github.com/dbt-labs/dbt-redshift.git git+https://github.com/dbt-labs/dbt-snowflake.git git+https://github.com/dbt-labs/dbt-bigquery.git +pytest-xdist diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index f115b4e2..f258464c 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,99 +1,67 @@ +{# These macros have been moved into dbt-core #} +{# Here for backwards compatibility ONLY #} + {# string ------------------------------------------------- #} {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} + {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_string() %} - string + {{ return(adapter.dispatch('type_string', 'dbt')()) }} {% endmacro %} -{%- macro redshift__type_string() -%} - varchar -{%- endmacro -%} - -{% macro postgres__type_string() %} - varchar -{% endmacro %} - -{% macro snowflake__type_string() %} - varchar -{% endmacro %} - - {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} + {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_timestamp() %} - timestamp -{% endmacro %} - -{% macro postgres__type_timestamp() %} - timestamp without time zone -{% endmacro %} - -{% macro snowflake__type_timestamp() %} - timestamp_ntz + {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }} {% endmacro %} {# float ------------------------------------------------- #} {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} + {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_float() %} - float + {{ return(adapter.dispatch('type_float', 'dbt')()) }} {% endmacro %} -{% macro bigquery__type_float() %} - float64 -{% endmacro %} {# numeric ------------------------------------------------ #} {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} + {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_numeric() %} - numeric(28, 6) -{% endmacro %} - -{% macro bigquery__type_numeric() %} - numeric + {{ return(adapter.dispatch('type_numeric', 'dbt')()) }} {% endmacro %} {# bigint ------------------------------------------------- #} {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} + {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_bigint() %} - bigint + {{ return(adapter.dispatch('type_bigint', 'dbt')()) }} {% endmacro %} -{% macro bigquery__type_bigint() %} - int64 -{% endmacro %} {# int ------------------------------------------------- #} {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} + {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} {%- endmacro -%} {% macro default__type_int() %} - int -{% endmacro %} - -{% macro bigquery__type_int() %} - int64 + {{ return(adapter.dispatch('type_int', 'dbt')()) }} {% endmacro %} diff --git a/run_functional_test.sh b/run_functional_test.sh index 4f8d0686..172d5aec 100755 --- a/run_functional_test.sh +++ b/run_functional_test.sh @@ -1,3 +1,3 @@ #!/bin/bash -python3 -m pytest tests/functional --profile $1 +python3 -m pytest tests/functional -n4 --profile $1 diff --git a/tests/conftest.py b/tests/conftest.py index 4858b21f..e7d9d198 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,10 +5,10 @@ def pytest_addoption(parser): - parser.addoption("--profile", action="store", default="apache_spark", type=str) + parser.addoption("--profile", action="store", default="postgres", type=str) -# Using @pytest.mark.skip_adapter('apache_spark') uses the 'skip_by_adapter_type' +# Using @pytest.mark.skip_profile('postgres') uses the 'skip_by_profile_type' # autouse fixture below def pytest_configure(config): config.addinivalue_line( diff --git a/tests/functional/cross_db_utils/base_cross_db_macro.py b/tests/functional/cross_db_utils/base_cross_db_macro.py deleted file mode 100644 index a504aaf5..00000000 --- a/tests/functional/cross_db_utils/base_cross_db_macro.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import pytest -from dbt.tests.util import run_dbt -from tests.functional.cross_db_utils.fixture_cross_db_macro import ( - macros__test_assert_equal_sql, -) - - -class BaseCrossDbMacro: - # install this repo as a package! - @pytest.fixture(scope="class") - def packages(self): - return {"packages": [{"local": os.getcwd()}]} - - # setup - @pytest.fixture(scope="class") - def macros(self): - return {"test_assert_equal.sql": macros__test_assert_equal_sql} - - # each child class will reimplement 'models' + 'seeds' - def seeds(self): - return {} - - def models(self): - return {} - - # actual test sequence - def test_build_assert_equal(self, project): - run_dbt(['deps']) - run_dbt(['build']) # seed, model, test -- all handled by dbt diff --git a/tests/functional/cross_db_utils/fixture_cross_db_macro.py b/tests/functional/cross_db_utils/fixture_cross_db_macro.py deleted file mode 100644 index 76fdb0fb..00000000 --- a/tests/functional/cross_db_utils/fixture_cross_db_macro.py +++ /dev/null @@ -1,6 +0,0 @@ -macros__test_assert_equal_sql = """ -{% test assert_equal(model, actual, expected) %} -select * from {{ model }} where {{ actual }} != {{ expected }} - -{% endtest %} -""" diff --git a/tests/functional/data_type/base_data_type.py b/tests/functional/data_type/base_data_type.py new file mode 100644 index 00000000..5bd0ceaa --- /dev/null +++ b/tests/functional/data_type/base_data_type.py @@ -0,0 +1,46 @@ +import os +import pytest +from dbt.tests.util import run_dbt, check_relations_equal, get_relation_columns +from dbt.tests.adapter.utils.data_types.base_data_type_macro import BaseDataTypeMacro + +class BaseDbtUtilsBackCompat(BaseDataTypeMacro): + # install this repo as a package + @pytest.fixture(scope="class") + def packages(self): + return {"packages": [{"local": os.getcwd()}]} + + # call the macros from the 'dbt_utils' namespace + # instead of the unspecified / global namespace + def macro_namespace(self): + return "dbt_utils" + + # actual test sequence needs to run 'deps' first + def test_check_types_assert_match(self, project): + run_dbt(['deps']) + super().test_check_types_assert_match(project) + + +class BaseLegacyDataTypeMacro(BaseDbtUtilsBackCompat): + def assert_columns_equal(self, project, expected_cols, actual_cols): + # we need to be a little more lenient when mapping between 'legacy' and 'new' types that are equivalent + # e.g. 'character varying' and 'text' + if expected_cols == actual_cols: + # cool, no need for jank + pass + else: + # this is pretty janky + # our goal here: reasonable confidence that the switch from the legacy version of the dbt_utils.type_{X} macro, + # and the new version, will not constitute a breaking change for end users + for (expected_col, actual_col) in zip(expected_cols, actual_cols): + expected = project.adapter.Column(*expected_col) + actual = project.adapter.Column(*actual_col) + print(f"Subtle type difference detected: {expected.data_type} vs. {actual.data_type}") + if any(( + expected.is_string() and actual.is_string(), + expected.is_float() and actual.is_float(), + expected.is_integer() and actual.is_integer(), + expected.is_numeric() and actual.is_numeric(), + )): + pytest.xfail() + else: + pytest.fail() diff --git a/tests/functional/data_type/test_type_bigint.py b/tests/functional/data_type/test_type_bigint.py new file mode 100644 index 00000000..f1f54742 --- /dev/null +++ b/tests/functional/data_type/test_type_bigint.py @@ -0,0 +1,26 @@ +import pytest +from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro +from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt + + +class TestTypeBigInt(BaseDbtUtilsBackCompat, BaseTypeBigInt): + pass + + +# previous dbt_utils code +macros__legacy_sql = """ +{% macro default__type_bigint() %} + bigint +{% endmacro %} + +{% macro bigquery__type_bigint() %} + int64 +{% endmacro %} +""" + +class TestTypeBigIntLegacy(BaseLegacyDataTypeMacro, BaseTypeBigInt): + @pytest.fixture(scope="class") + def macros(self): + return { + "legacy.sql": macros__legacy_sql + } diff --git a/tests/functional/data_type/test_type_float.py b/tests/functional/data_type/test_type_float.py new file mode 100644 index 00000000..8afd68e0 --- /dev/null +++ b/tests/functional/data_type/test_type_float.py @@ -0,0 +1,27 @@ +import pytest +from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro +from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat + + +class TestTypeFloat(BaseDbtUtilsBackCompat, BaseTypeFloat): + pass + + +# previous dbt_utils code +macros__legacy_sql = """ +{% macro default__type_float() %} + float +{% endmacro %} + +{% macro bigquery__type_float() %} + float64 +{% endmacro %} +""" + + +class TestTypeFloatLegacy(BaseLegacyDataTypeMacro, BaseTypeFloat): + @pytest.fixture(scope="class") + def macros(self): + return { + "legacy.sql": macros__legacy_sql + } diff --git a/tests/functional/data_type/test_type_int.py b/tests/functional/data_type/test_type_int.py new file mode 100644 index 00000000..97e74c41 --- /dev/null +++ b/tests/functional/data_type/test_type_int.py @@ -0,0 +1,27 @@ +import pytest +from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro +from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt + + +class TestTypeInt(BaseDbtUtilsBackCompat, BaseTypeInt): + pass + + +# previous dbt_utils code +macros__legacy_sql = """ +{% macro default__type_int() %} + int +{% endmacro %} + +{% macro bigquery__type_int() %} + int64 +{% endmacro %} +""" + + +class TestTypeFloatLegacy(BaseLegacyDataTypeMacro, BaseTypeInt): + @pytest.fixture(scope="class") + def macros(self): + return { + "legacy.sql": macros__legacy_sql + } diff --git a/tests/functional/data_type/test_type_numeric.py b/tests/functional/data_type/test_type_numeric.py new file mode 100644 index 00000000..ebbc4db3 --- /dev/null +++ b/tests/functional/data_type/test_type_numeric.py @@ -0,0 +1,49 @@ +import pytest +from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro +from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric + + +@pytest.mark.skip_profile('bigquery') +class TestTypeNumeric(BaseDbtUtilsBackCompat, BaseTypeNumeric): + pass + + +@pytest.mark.only_profile('bigquery') +class TestBigQueryTypeNumeric(BaseDbtUtilsBackCompat, BaseTypeNumeric): + def numeric_fixture_type(self): + return "numeric" + + +# previous dbt_utils code +macros__legacy_sql = """ +{% macro default__type_numeric() %} + numeric(28, 6) +{% endmacro %} +{% macro bigquery__type_numeric() %} + numeric +{% endmacro %} +""" + + +class BaseTypeNumericLegacy(BaseLegacyDataTypeMacro, BaseTypeNumeric): + @pytest.fixture(scope="class") + def macros(self): + return { + "legacy.sql": macros__legacy_sql + } + + +@pytest.mark.skip_profile('bigquery') +class TestTypeNumeric(BaseTypeNumeric): + pass + + +@pytest.mark.skip_profile('bigquery') +class TestTypeNumericLegacy(BaseTypeNumericLegacy): + pass + + +@pytest.mark.only_profile('bigquery') +class TestBigQueryTypeNumericLegacy(BaseTypeNumericLegacy): + def numeric_fixture_type(self): + return "numeric" diff --git a/tests/functional/data_type/test_type_string.py b/tests/functional/data_type/test_type_string.py new file mode 100644 index 00000000..64b2b81e --- /dev/null +++ b/tests/functional/data_type/test_type_string.py @@ -0,0 +1,35 @@ +import pytest +from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro +from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString + + +class TestTypeInt(BaseDbtUtilsBackCompat, BaseTypeString): + pass + + +# previous dbt_utils code +macros__legacy_sql = """ +{% macro default__type_string() %} + string +{% endmacro %} + +{%- macro redshift__type_string() -%} + varchar +{%- endmacro -%} + +{% macro postgres__type_string() %} + varchar +{% endmacro %} + +{% macro snowflake__type_string() %} + varchar +{% endmacro %} +""" + + +class TestTypeStringLegacy(BaseLegacyDataTypeMacro, BaseTypeString): + @pytest.fixture(scope="class") + def macros(self): + return { + "legacy.sql": macros__legacy_sql + } diff --git a/tests/functional/data_type/test_type_timestamp.py b/tests/functional/data_type/test_type_timestamp.py new file mode 100644 index 00000000..da87ce4b --- /dev/null +++ b/tests/functional/data_type/test_type_timestamp.py @@ -0,0 +1,31 @@ +import pytest +from tests.functional.data_type.base_data_type import BaseDbtUtilsBackCompat, BaseLegacyDataTypeMacro +from dbt.tests.adapter.utils.data_types.test_type_timestamp import BaseTypeTimestamp + + +class TestTypeTimestamp(BaseDbtUtilsBackCompat, BaseTypeTimestamp): + pass + + +# previous dbt_utils code +macros__legacy_sql = """ +{% macro default__type_timestamp() %} + timestamp +{% endmacro %} + +{% macro postgres__type_timestamp() %} + timestamp without time zone +{% endmacro %} + +{% macro snowflake__type_timestamp() %} + timestamp_ntz +{% endmacro %} +""" + + +class TestTypeTimestampLegacy(BaseLegacyDataTypeMacro, BaseTypeTimestamp): + @pytest.fixture(scope="class") + def macros(self): + return { + "legacy.sql": macros__legacy_sql + }