From c66b43a0a3077c247874cf6ba27693402bda4a65 Mon Sep 17 00:00:00 2001 From: Adolfo Rodriguez Date: Fri, 16 Feb 2024 07:45:22 -0800 Subject: [PATCH 1/5] Escape upstream descriptions in generate_model_yaml --- CHANGELOG.md | 3 +++ integration_tests/models/schema.yml | 2 +- .../tests/test_generate_model_yaml_upstream_descriptions.sql | 2 +- macros/generate_model_yaml.sql | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e1cd0b..278538d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## Fixes +- Column `description` fields are now correctly escaped in `generate_model_yaml` ([#142](https://github.com/dbt-labs/dbt-codegen/issues/142)) + # dbt-codegen v0.11.0 ## 🚨 Breaking change diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml index aa3f980..4fe5ae7 100644 --- a/integration_tests/models/schema.yml +++ b/integration_tests/models/schema.yml @@ -4,4 +4,4 @@ models: - name: model_data_a columns: - name: col_a - description: description column a \ No newline at end of file + description: 'description column "a"' \ No newline at end of file diff --git a/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql b/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql index 73e6687..613345a 100644 --- a/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql +++ b/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql @@ -13,7 +13,7 @@ models: description: "" columns: - name: col_a - description: "description column a" + description: "description column \"a\"" - name: col_b description: "" diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index d10e9a4..d4fbbee 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -13,7 +13,7 @@ {% if include_data_types %} {% do model_yaml.append(' data_type: ' ~ codegen.data_type_format_model(column)) %} {% endif %} - {% do model_yaml.append(' description: "' ~ column_desc_dict.get(column.name | lower,'') ~ '"') %} + {% do model_yaml.append(' description: ' ~ (column_desc_dict.get(column.name | lower,'') | tojson)) %} {% do model_yaml.append('') %} {% if column.fields|length > 0 %} From e7a71c6c01effeccfdaaf35489d30ab69d1cad6e Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 28 Feb 2024 14:56:36 -0600 Subject: [PATCH 2/5] Add config to implement late binding views --- dbt_project.yml | 7 +++++-- integration_tests/package-lock.yml | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 integration_tests/package-lock.yml diff --git a/dbt_project.yml b/dbt_project.yml index 91dfcf1..0d0cc05 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ -name: 'codegen' -version: '0.5.0' +name: "codegen" +version: "0.5.0" require-dbt-version: [">=1.1.0", "<2.0.0"] config-version: 2 @@ -8,3 +8,6 @@ target-path: "target" clean-targets: ["target", "dbt_packages"] macro-paths: ["macros"] log-path: "logs" + +models: + +bind: false diff --git a/integration_tests/package-lock.yml b/integration_tests/package-lock.yml new file mode 100644 index 0000000..88d613b --- /dev/null +++ b/integration_tests/package-lock.yml @@ -0,0 +1,5 @@ +packages: +- local: ../ +- package: dbt-labs/dbt_utils + version: 1.1.1 +sha1_hash: de2deba3d66ce03d8c02949013650cc9b94f6030 From 528cfae3d43be9c860abf1b595a953e3c73039b5 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 28 Feb 2024 15:34:27 -0600 Subject: [PATCH 3/5] Add config to implement late binding views IN THE TEST PACKAGE --- dbt_project.yml | 3 --- integration_tests/dbt_project.yml | 16 ++++++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 0d0cc05..4ee4e03 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -8,6 +8,3 @@ target-path: "target" clean-targets: ["target", "dbt_packages"] macro-paths: ["macros"] log-path: "logs" - -models: - +bind: false diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 22a90e8..db3781e 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,8 +1,8 @@ -name: 'codegen_integration_tests' -version: '1.0' +name: "codegen_integration_tests" +version: "1.0" config-version: 2 -profile: 'integration_tests' +profile: "integration_tests" model-paths: ["models"] analysis-paths: ["analyses"] @@ -12,12 +12,16 @@ macro-paths: ["macros"] target-path: "target" clean-targets: - - "target" - - "dbt_packages" + - "target" + - "dbt_packages" seeds: +schema: raw_data +quote_columns: false vars: - my_table_reference: table_c \ No newline at end of file + my_table_reference: table_c + +models: + +bind: false + From d08323fdfd8e298d26de3a57f94e87c34004e8d3 Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 28 Feb 2024 15:43:53 -0600 Subject: [PATCH 4/5] Add cascade to source table create operation table drops --- integration_tests/macros/operations/create_source_table.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/macros/operations/create_source_table.sql b/integration_tests/macros/operations/create_source_table.sql index 7dad625..8c3902a 100644 --- a/integration_tests/macros/operations/create_source_table.sql +++ b/integration_tests/macros/operations/create_source_table.sql @@ -16,7 +16,7 @@ reset enable_case_sensitive_identifier; {% do adapter.create_schema(target_schema) %} {% set drop_table_sql %} -drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table +drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table cascade {% endset %} {{ run_query(drop_table_sql) }} @@ -33,7 +33,7 @@ create table {{ target_schema }}.codegen_integration_tests__data_source_table as {{ run_query(create_table_sql) }} {% set drop_table_sql_case_sensitive %} -drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive +drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive cascade {% endset %} {{ run_query(drop_table_sql_case_sensitive) }} From 0f07873c26ddbcbed9e26a964f8b9c4220fdb38e Mon Sep 17 00:00:00 2001 From: gwen windflower Date: Wed, 28 Feb 2024 15:50:14 -0600 Subject: [PATCH 5/5] Make cascade drop redshift only --- integration_tests/macros/operations/create_source_table.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/macros/operations/create_source_table.sql b/integration_tests/macros/operations/create_source_table.sql index 8c3902a..6794e51 100644 --- a/integration_tests/macros/operations/create_source_table.sql +++ b/integration_tests/macros/operations/create_source_table.sql @@ -16,7 +16,7 @@ reset enable_case_sensitive_identifier; {% do adapter.create_schema(target_schema) %} {% set drop_table_sql %} -drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table cascade +drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table {% if target.type == "redshift" %}cascade{% endif %} {% endset %} {{ run_query(drop_table_sql) }} @@ -33,7 +33,7 @@ create table {{ target_schema }}.codegen_integration_tests__data_source_table as {{ run_query(create_table_sql) }} {% set drop_table_sql_case_sensitive %} -drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive cascade +drop table if exists {{ target_schema }}.codegen_integration_tests__data_source_table_case_sensitive {% if target.type == "redshift" %}cascade{% endif %} {% endset %} {{ run_query(drop_table_sql_case_sensitive) }}