From 8c1cf596d35f40775309272268982ea188e1b0fa Mon Sep 17 00:00:00 2001 From: Grigorii Date: Wed, 21 Dec 2022 03:31:09 +0200 Subject: [PATCH] Add include_data_types flag to generate_source macro (#76) * Add include_data_types flag to generate_source macro * Add test for include_data_types * Fix test_generate_source_all_args for Redshift * Fix test_generate_source_all_args for Snowflake * Fix int types test_generate_source_all_args for Snowflake * Fix test_generate_source_all_args for BigQuery * Fix typo * Fix typo * trying to avoid db-specific type defenitions * Dummy fix * Dummy fix * Dummy fix * Dummy fix * Revert to initial schema I've tried several approaches to generate the exact text, but it's not working. So returninig tests to https://github.com/dbt-labs/dbt-codegen/pull/76/commits/3a1267bd1511d7fa2f200d606a7ce1fea515156f state. * Example of generating data types for columns * Update changelog * Remove comment [skip ci] Co-authored-by: Doug Beatty --- CHANGELOG.md | 2 ++ README.md | 10 +++++++++- integration_tests/macros/integer_type_value.sql | 9 +++++++++ integration_tests/macros/text_type_value.sql | 11 +++++++++++ .../tests/test_generate_source_all_args.sql | 10 ++++++++++ macros/generate_source.sql | 5 ++++- 6 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 integration_tests/macros/integer_type_value.sql create mode 100644 integration_tests/macros/text_type_value.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b1c72d..6cdee1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ## New features - Addition of the [create_base_models](macros/create_base_models.sql) This macro generates a series of terminal commands (appended w) bash script which creates a new file in your dbt project based off the results of the [generate_base_model](macros/generate_base_model.sql) macro. Therefore, instead of outputting in the terminal, it will create the file for you. +- Add `include_data_types` flag to `generate_source` macro ([#76](https://github.com/dbt-labs/dbt-codegen/pull/76)) ## Quality of life - Addition of the [base_model_creation](bash_scripts/base_model_creation.sh) bash script which allows users to input multiple tables as a list and generate a terminal command that will combine **all** [create_base_models](macros/create_base_models.sql) commands. This way, you can generate base models for all your sources at once. @@ -21,6 +22,7 @@ This macro generates a series of terminal commands (appended w) bash script whic ## Contributors: - [@fivetran-joemarkiewicz](https://github.com/fivetran-joemarkiewicz) (#83) +- [@GSokol](https://github.com/GSokol) (#76) # dbt-codegen v0.9.0 diff --git a/README.md b/README.md index e082cd2..cf70350 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,9 @@ source data is in. column names to your source definition. * `include_descriptions` (optional, default=False): Whether you want to add description placeholders to your source definition. -* `table_pattern` (optional, default='%'): A table prefix / postfix that you +* `include_data_types` (optional, default=False): Whether you want to add data +types to your source columns definitions. +* `table_pattern` (optional, default='%'): A table prefix / postfix that you want to subselect from all available tables within a given schema. * `exclude` (optional, default=''): A string you want to exclude from the selection criteria * `name` (optional, default=schema_name): The name of your source @@ -75,6 +77,12 @@ or $ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "database_name": "raw", "table_names":["table_1", "table_2"]}' ``` +Including data types: + +``` +$ dbt run-operation generate_source --args '{"schema_name": "jaffle_shop", "generate_columns": "true", "include_data_types": "true"}' +``` + 2. The YAML for the source will be logged to the command line ``` diff --git a/integration_tests/macros/integer_type_value.sql b/integration_tests/macros/integer_type_value.sql new file mode 100644 index 0000000..f447110 --- /dev/null +++ b/integration_tests/macros/integer_type_value.sql @@ -0,0 +1,9 @@ +{%- macro integer_type_value() -%} +{%- if target.type == "snowflake" -%} +NUMBER(38,0) +{%- elif target.type == "bigquery" -%} +INT64 +{%- else -%} +INTEGER +{%- endif -%} +{%- endmacro -%} diff --git a/integration_tests/macros/text_type_value.sql b/integration_tests/macros/text_type_value.sql new file mode 100644 index 0000000..929813f --- /dev/null +++ b/integration_tests/macros/text_type_value.sql @@ -0,0 +1,11 @@ +{%- macro text_type_value(text_length) -%} +{%- if target.type == "redshift" -%} +CHARACTER VARYING({{ text_length }}) +{%- elif target.type == "snowflake" -%} +CHARACTER VARYING(16777216) +{%- elif target.type == "bigquery" -%} +STRING +{%- else -%} +TEXT +{%- endif -%} +{%- endmacro -%} diff --git a/integration_tests/tests/test_generate_source_all_args.sql b/integration_tests/tests/test_generate_source_all_args.sql index e60faf8..fb46970 100644 --- a/integration_tests/tests/test_generate_source_all_args.sql +++ b/integration_tests/tests/test_generate_source_all_args.sql @@ -8,6 +8,7 @@ database_name=target.database, generate_columns=True, include_descriptions=True, + include_data_types=True, name=raw_schema ) %} @@ -24,30 +25,39 @@ sources: description: "" columns: - name: col_a + data_type: {{ integer_type_value() }} description: "" - name: col_b + data_type: {{ text_type_value(1) }} description: "" - name: data__b_relation description: "" columns: - name: col_a + data_type: {{ integer_type_value() }} description: "" - name: col_b + data_type: {{ text_type_value(1) }} description: "" - name: data__campaign_analytics description: "" columns: - name: source + data_type: {{ text_type_value(8) }} description: "" - name: medium + data_type: {{ text_type_value(8) }} description: "" - name: source_medium + data_type: {{ text_type_value(2) }} description: "" - name: analytics + data_type: {{ integer_type_value() }} description: "" - name: col_x + data_type: {{ text_type_value(1) }} description: "" {% endset %} diff --git a/macros/generate_source.sql b/macros/generate_source.sql index 366b4b8..df81053 100644 --- a/macros/generate_source.sql +++ b/macros/generate_source.sql @@ -15,7 +15,7 @@ --- -{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, table_pattern='%', exclude='', name=schema_name, table_names=None) %} +{% macro generate_source(schema_name, database_name=target.database, generate_columns=False, include_descriptions=False, include_data_types=False, table_pattern='%', exclude='', name=schema_name, table_names=None) %} {% set sources_yaml=[] %} {% do sources_yaml.append('version: 2') %} @@ -61,6 +61,9 @@ {% for column in columns %} {% do sources_yaml.append(' - name: ' ~ column.name | lower ) %} + {% if include_data_types %} + {% do sources_yaml.append(' data_type: ' ~ (column.data_type | upper ) ) %} + {% endif %} {% if include_descriptions %} {% do sources_yaml.append(' description: ""' ) %} {% endif %}