From b63e433895c375e09504d16aa7d0e4387ee9eb3a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:15:09 -0800 Subject: [PATCH 01/11] initialize and point to working utils branch --- packages.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages.yml b/packages.yml index c71e87a5..24690ce7 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,9 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] - +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +# - local: ../dbt_fivetran_utils +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: feature/sql-server-compatibility + warn-unpinned: false - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] From 2d5d0a11cea580fcf607f826d0807b6e67211527 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:40:02 -0800 Subject: [PATCH 02/11] deprecation city --- CHANGELOG.md | 5 ++ README.md | 12 +-- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 16 +--- integration_tests/seeds/transformation.csv | 3 - integration_tests/seeds/trigger_table.csv | 3 - .../seeds/trigger_table_snowflake.csv | 3 - models/fivetran_platform.yml | 56 -------------- ...vetran_platform__transformation_status.sql | 77 ------------------- models/staging/src_fivetran_platform.yml | 51 ------------ models/staging/stg_fivetran_platform.yml | 47 ----------- .../stg_fivetran_platform__transformation.sql | 27 ------- .../stg_fivetran_platform__trigger_table.sql | 25 ------ 13 files changed, 9 insertions(+), 318 deletions(-) delete mode 100644 integration_tests/seeds/transformation.csv delete mode 100644 integration_tests/seeds/trigger_table.csv delete mode 100644 integration_tests/seeds/trigger_table_snowflake.csv delete mode 100644 models/fivetran_platform__transformation_status.sql delete mode 100644 models/staging/stg_fivetran_platform__transformation.sql delete mode 100644 models/staging/stg_fivetran_platform__trigger_table.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a373740..ef9d9162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_fivetran_log v1.3.0 + +## Breaking Changes +- Deprecated the `transformation` and `trigger_table` source tables and any downstream transforms. These tables only housed information on Fivetran Basic SQL Transformations, which were sunset last year. + # dbt_fivetran_log v1.2.0 [PR #92](https://github.com/fivetran/dbt_fivetran_log/pull/92) includes the following updates: diff --git a/README.md b/README.md index 5ee90234..d7923ecc 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ # 📣 What does this dbt package do? - Generates a comprehensive data dictionary of your Fivetran Platform connector (previously called Fivetran Log) data via the [dbt docs site](https://fivetran.github.io/dbt_fivetran_log/) - Produces staging models in the format described by [this ERD](https://fivetran.com/docs/logs/fivetran-log#schemainformation) which clean, test, and prepare your Fivetran data from [Fivetran's free connector](https://fivetran.com/docs/applications/fivetran-log) and generates analysis ready end models. -- The above mentioned models enable you to better understand how you are spending money in Fivetran according to our [consumption-based pricing model](https://fivetran.com/docs/getting-started/consumption-based-pricing) as well as providing details about the performance and status of your Fivetran connectors and transformations. This is achieved by: +- The above mentioned models enable you to better understand how you are spending money in Fivetran according to our [consumption-based pricing model](https://fivetran.com/docs/getting-started/consumption-based-pricing) as well as providing details about the performance and status of your Fivetran connectors. This is achieved by: - Displaying consumption data at the table, connector, destination, and account levels - Providing a history of measured free and paid monthly active rows (MAR), credit consumption, and the relationship between the two - Creating a history of vital daily events for each connector @@ -30,7 +30,6 @@ Refer to the table below for a detailed view of all models materialized by defau | **model** | **description** | | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [fivetran_platform__connector_status](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__connector_status) | Each record represents a connector loading data into a destination, enriched with data about the connector's data sync status. | -| [fivetran_platform__transformation_status](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__transformation_status) | Each record represents a transformation, enriched with data about the transformation's last sync and any tables whose new data triggers the transformation to run. | | [fivetran_platform__mar_table_history](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__mar_table_history) | Each record represents a table's free, paid, and total volume for a month, complete with data about its connector and destination. | | [fivetran_platform__usage_mar_destination_history](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__usage_mar_destination_history) | Table of each destination's usage and active volume, per month. Includes the usage per million MAR and MAR per usage. Usage either refers to a dollar or credit amount, depending on customer's pricing model. Read more about the relationship between usage and MAR [here](https://www.fivetran.com/legal/service-consumption-table). | | [fivetran_platform__connector_daily_events](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__connector_daily_events) | Each record represents a daily measurement of the API calls, schema changes, and record modifications made by a connector, starting from the date on which the connector was set up. | @@ -81,20 +80,15 @@ vars: ``` ## Step 4: Disable Models for Non Existent Sources -If you have never created Fivetran-orchestrated [basic SQL transformations](https://fivetran.com/docs/transformations/basic-sql), your source data will not contain the `transformation` and `trigger_table` tables. Moreover, if you have only created *scheduled* basic transformations that are not triggered by table syncs, your source data will not contain the `trigger_table` table (though it will contain `transformation`). - -Additionally, if you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` sources. To disable the corresponding functionality in the package, you must add the following variable(s) to your root `dbt_project.yml` file. By default, all variables are assumed to be `true`: +If you do not leverage Fivetran RBAC, then you will not have the `user` or `destination_membership` sources. To disable the corresponding functionality in the package, you must add the following variable(s) to your root `dbt_project.yml` file. By default, all variables are assumed to be `true`: ```yml vars: - fivetran_platform_using_transformations: false # this will disable all transformation + trigger_table logic - fivetran_platform_using_triggers: false # this will disable only trigger_table logic fivetran_platform_using_destination_membership: false # this will disable only the destination membership logic fivetran_platform_using_user: false # this will disable only the user logic ``` ## (Optional) Step 5: Additional Configurations -
Expand for configurations ### Configuring Fivetran Error and Warning Messages Some users may wish to exclude Fivetran error and warnings messages from the final `fivetran_platform__connector_status` model due to the length of the message. To disable the `errors_since_last_completed_sync` and `warnings_since_last_completed_sync` fields from the final model you may add the following variable to you to your root `dbt_project.yml` file. By default, this variable is assumed to be `true`: @@ -129,8 +123,6 @@ dispatch: search_order: ['spark_utils', 'dbt_utils'] ``` -
- ## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™
Expand for details
diff --git a/dbt_project.yml b/dbt_project.yml index 7efde40c..2fac2cab 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ config-version: 2 name: 'fivetran_log' -version: '1.2.0' +version: '1.3.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4069d7cc..787826e4 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'fivetran_log_integration_tests' -version: '1.2.0' +version: '1.3.0' config-version: 2 profile: 'integration_tests' @@ -19,8 +19,6 @@ vars: fivetran_platform_destination_identifier: "destination" fivetran_platform_destination_membership_identifier: "destination_membership" fivetran_platform_log_identifier: "log" - fivetran_platform_transformation_identifier: "transformation" - trigger_table: "{{ fivetran_utils.snowflake_seed_data('trigger_table') }}" fivetran_platform_user_identifier: "user" @@ -57,18 +55,6 @@ seeds: +column_types: time_stamp: timestamp transformation_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - transformation: - +column_types: - created_at: timestamp - destination_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - trigger_table: - +quote_columns: "{{ true if target.type in ('redshift', 'postgres') else false }}" - +enabled: "{{ true if target.type != 'snowflake' else false }}" - +column_types: - transformation_id: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - trigger_table_snowflake: - +enabled: "{{ true if target.type == 'snowflake' else false }}" user: +column_types: created_at: timestamp diff --git a/integration_tests/seeds/transformation.csv b/integration_tests/seeds/transformation.csv deleted file mode 100644 index e918a105..00000000 --- a/integration_tests/seeds/transformation.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,_fivetran_synced,created_at,created_by_id,destination_id,name,paused,script,trigger_delay,trigger_interval,trigger_type -nullify_emotion,2021-03-12 21:03:46.301,2020-11-19 20:14:34.925,tome_typing,1k56c2c4xlti6,test_once_per_day_time,false,select 1;,,,DAILY -footman_cone,2021-03-03 15:08:47.778,2020-11-25 17:40:11.403,tabby_wallet,unearned_garret,fivetran_log__Calculate_MAR_by_Schema,false,select 0;,,,DAILY diff --git a/integration_tests/seeds/trigger_table.csv b/integration_tests/seeds/trigger_table.csv deleted file mode 100644 index d397bb79..00000000 --- a/integration_tests/seeds/trigger_table.csv +++ /dev/null @@ -1,3 +0,0 @@ -table,transformation_id,_fivetran_synced -intercom.admin,benign_cultivate,2021-03-12 21:03:46.303 -salesforce.account,novelty_unite,2021-03-12 21:03:46.303 diff --git a/integration_tests/seeds/trigger_table_snowflake.csv b/integration_tests/seeds/trigger_table_snowflake.csv deleted file mode 100644 index 1605dbd1..00000000 --- a/integration_tests/seeds/trigger_table_snowflake.csv +++ /dev/null @@ -1,3 +0,0 @@ -"""TABLE""",transformation_id,_fivetran_synced -intercom.admin,benign_cultivate,2021-03-12 21:03:46.303 -salesforce.account,novelty_unite,2021-03-12 21:03:46.303 diff --git a/models/fivetran_platform.yml b/models/fivetran_platform.yml index fc6f0b1b..376ca653 100644 --- a/models/fivetran_platform.yml +++ b/models/fivetran_platform.yml @@ -167,62 +167,6 @@ models: - name: dollars_spent description: The dollar amount used by the destination for the given month. - - name: fivetran_platform__transformation_status - description: > - Table of transformations enriched with data regarding its run status and any tables - whose new data trigger the transformation to run. - - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - destination_id - - transformation_id - - columns: - - name: transformation_id - description: Unique ID of the transformation. - - - name: created_at - description: Timestamp of when the transformation was created. - - - name: created_by_user_id - description: Foreign key referencing the `user` who created the transformation. - - - name: destination_id - description: Foreign key referenencing the `destination` whose data is being transformed. - - - name: transformation_name - description: Name given to the transformation. - - - name: is_paused - description: Boolean of whether the data transformation is currently paused. - - - name: script - description: The code content of the transformation. - - - name: trigger_delay - description: The delay in triggering transformation. - - - name: trigger_interval - description: The scheduled frequency of the transformation, if a time schedule is chosen as its trigger. - - - name: trigger_type - description: The schedule type of the transformation (time schedule vs. new data in certain tables). - - - name: destination_name - description: Name of the destination in which the destination is run. - - - name: last_successful_run_at - description: Timestamp of when the transformation was last run successfully. - - - name: last_run_attempt - description: Outcome of the last run attempt (`"success"` or `"failure"`). - - - name: trigger_tables - description: > - Aggregated string list of the tables that trigger the transformation to run, if it is - not on a time schedule. - - name: fivetran_platform__connector_daily_events description: > Table of each connector's daily history of the quantity of api calls, schema changes, and records modified, diff --git a/models/fivetran_platform__transformation_status.sql b/models/fivetran_platform__transformation_status.sql deleted file mode 100644 index e6748aff..00000000 --- a/models/fivetran_platform__transformation_status.sql +++ /dev/null @@ -1,77 +0,0 @@ -{{ config(enabled=var('fivetran_platform_using_transformations', True)) }} - -with transformation as ( - - select * - from {{ ref('stg_fivetran_platform__transformation') }} -), - -destination as ( - - select * - from {{ ref('stg_fivetran_platform__destination') }} -), - -{% if var('fivetran_platform_using_triggers', True) %} -trigger_tables as ( - - select - transformation_id, - {{ fivetran_utils.string_agg("trigger_table", "', '") }} as trigger_tables - - from {{ ref('stg_fivetran_platform__trigger_table') }} - - group by 1 -), -{% endif %} - -transformation_runs as ( - - select * - from {{ ref('stg_fivetran_platform__log') }} - - where event_subtype = 'transformation run success' - or event_subtype = 'transformation run failed' -), - -recent_runs as ( - - select - transformation_id, - max(case when event_subtype = 'transformation run success' then created_at else null end) as last_run_at, - max(case when event_subtype = 'transformation run failed' then created_at else null end) as last_failure_at - - from transformation_runs - group by 1 -), - -transformation_join as ( - - select - transformation.*, - destination.destination_name, - recent_runs.last_run_at as last_successful_run_at, - case when recent_runs.last_run_at > recent_runs.last_failure_at or recent_runs.last_failure_at is null then 'success' - else 'failure' end as last_run_attempt - {% if var('fivetran_platform_using_triggers', True) %} - , - trigger_tables.trigger_tables - {% endif %} - - from - transformation - join destination - on destination.destination_id = transformation.destination_id - - {% if var('fivetran_platform_using_triggers', True) %} - left join trigger_tables - on trigger_tables.transformation_id = transformation.transformation_id - {% endif %} - - left join recent_runs - on recent_runs.transformation_id = transformation.transformation_id - -) - - -select * from transformation_join \ No newline at end of file diff --git a/models/staging/src_fivetran_platform.yml b/models/staging/src_fivetran_platform.yml index a83b1a32..ad8fa146 100644 --- a/models/staging/src_fivetran_platform.yml +++ b/models/staging/src_fivetran_platform.yml @@ -198,56 +198,5 @@ sources: description: Phone number associated with user. - name: verified description: Boolean that indicates whether the user has verified their email address in the account creation process. - - name: _fivetran_synced - description: Timestamp of when the record was last synced. - - - name: trigger_table - identifier: "{{ var('fivetran_platform_trigger_table_identifier', 'trigger_table')}}" - description: > - Table relating transformations to the connector source tables that trigger them. - Transformations on time schedules do not appear here. Note - this table is only created - once you have created a triggered transformation. - config: - enabled: "{{ var('fivetran_platform_using_triggers', true) }}" - freshness: null - # transformation data tests are currently implemented at the staging level to accommodate schemas without this source table - columns: - - name: table - description: Connector source table whose new data triggers the transformation. - - name: transformation_id - description: Foreign key referencing the `transformation` that is triggered by new data. - - name: _fivetran_synced - description: Timestamp of when the record was last synced. - - - name: transformation - identifier: "{{ var('fivetran_platform_transformation_identifier', 'transformation')}}" - description: > - Table of [transformations](https://fivetran.com/docs/transformations) made and carried out in Fivetran. - Note - this table is only created once you have created a transformation for the destination. - config: - enabled: "{{ var('fivetran_platform_using_transformations', true) }}" - freshness: null - # transformation data tests are currently implemented at the staging level to accommodate schemas without this source table - columns: - - name: id - description: Unique ID of the transformation. - - name: created_at - description: Timestamp of when the transformation was created. - - name: created_by_id - description: Foreign key referencing the `user` who created the transformation. - - name: destination_id - description: Foreign key referencing the `destination` whose data is being transformed. - - name: name - description: Name given to the transformation. - - name: paused - description: Boolean of whether the transformation is currently paused. - - name: script - description: The code content of the transformation. - - name: trigger_delay - description: The delay in triggering transformation. - - name: trigger_interval - description: The scheduled frequency of the transformation, if a time schedule is chosen as its trigger. - - name: trigger_type - description: The schedule type of the transformation (time schedule vs. new data in certain tables). - name: _fivetran_synced description: Timestamp of when the record was last synced. \ No newline at end of file diff --git a/models/staging/stg_fivetran_platform.yml b/models/staging/stg_fivetran_platform.yml index df6765f5..52501641 100644 --- a/models/staging/stg_fivetran_platform.yml +++ b/models/staging/stg_fivetran_platform.yml @@ -165,53 +165,6 @@ models: description: Timestamp when the user was added to the destination. - name: destination_role description: The user's permissions for this individual destination. - - - name: stg_fivetran_platform__transformation - description: > - Table of [transformations](https://fivetran.com/docs/transformations) made and carried out in Fivetran. - Note - this table is only created once you have created a transformation for the destination. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - transformation_id - - destination_id - columns: - - name: transformation_id - description: Destination-unique ID of the transformation. - - name: created_at - description: Timestamp of when the transformation was created. - - name: created_by_user_id - description: Foreign key referencing the `user` who created the transformation. - - name: destination_id - description: Foreign key referenencing the `destination` whose data is being transformed. - - name: transformation_name - description: Name given to the transformation. - - name: is_paused - description: Boolean of whether the data transformation is currently paused. - - name: script - description: The code content of the transformation. - - name: trigger_delay - description: The delay in triggering transformation. - - name: trigger_interval - description: The scheduled frequency of the transformation, if a time schedule is chosen as its trigger. - - name: trigger_type - description: The schedule type of the transformation (time schedule vs. new data in certain tables). - - - name: stg_fivetran_platform__trigger_table - description: > - Table relating transformations to the connector source tables that trigger them. - Transformations on time schedules do not appear here. Note - this table is only created - once you have created a triggered transformation. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - trigger_table - - transformation_id - columns: - - name: trigger_table - description: Connector source table whose new data triggers the transformation. - - name: transformation_id - description: Foreign key referencing the `transformation` that is triggered by new data. - name: stg_fivetran_platform__user description: Table of users given access to this Fivetran account. diff --git a/models/staging/stg_fivetran_platform__transformation.sql b/models/staging/stg_fivetran_platform__transformation.sql deleted file mode 100644 index 44a87936..00000000 --- a/models/staging/stg_fivetran_platform__transformation.sql +++ /dev/null @@ -1,27 +0,0 @@ -{{ config(enabled=var('fivetran_platform_using_transformations', True)) }} - -with base as ( - - select * - from {{ var('transformation') }} -), - -fields as ( - - select - id as transformation_id, - cast(created_at as {{ dbt.type_timestamp() }}) as created_at, - created_by_id as created_by_user_id, - destination_id, - name as transformation_name, - paused as is_paused, - script, - trigger_delay, - trigger_interval, - trigger_type - from base -) - -select * -from fields -where transformation_id is not null \ No newline at end of file diff --git a/models/staging/stg_fivetran_platform__trigger_table.sql b/models/staging/stg_fivetran_platform__trigger_table.sql deleted file mode 100644 index a1211e79..00000000 --- a/models/staging/stg_fivetran_platform__trigger_table.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config(enabled=var('fivetran_platform_using_triggers', True)) }} - -with base as ( - - select * - from {{ var('trigger_table') }} -), - -fields as ( - - select - {% if target.type == 'bigquery' %} - table as trigger_table, - {% elif target.type == 'postgres' %} - "table" as trigger_table, - {% else %} - "TABLE" as trigger_table, - {% endif %} - transformation_id - from base -) - -select * -from fields -where transformation_id is not null \ No newline at end of file From afd2b7cbac123f444bf66c51dfb6423ca74cd72d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:15:03 -0800 Subject: [PATCH 03/11] keep separated from sqlserver changes --- packages.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages.yml b/packages.yml index 24690ce7..908f4710 100644 --- a/packages.yml +++ b/packages.yml @@ -1,9 +1,5 @@ packages: -# - package: fivetran/fivetran_utils -# version: [">=0.4.0", "<0.5.0"] -# - local: ../dbt_fivetran_utils -- git: https://github.com/fivetran/dbt_fivetran_utils.git - revision: feature/sql-server-compatibility - warn-unpinned: false +- package: fivetran/fivetran_utils + version: [">=0.4.0", "<0.5.0"] - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] From cd004b01a2ef24e827cdc1dc12256b895fc2c16f Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:13:37 -0800 Subject: [PATCH 04/11] other stuff --- CHANGELOG.md | 2 +- integration_tests/requirements.txt | 3 +-- integration_tests/seeds/.DS_Store | Bin 6148 -> 6148 bytes 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef9d9162..48c4c132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_fivetran_log v1.3.0 ## Breaking Changes -- Deprecated the `transformation` and `trigger_table` source tables and any downstream transforms. These tables only housed information on Fivetran Basic SQL Transformations, which were sunset last year. +- Deprecated the `transformation` and `trigger_table` source tables and any downstream transforms. These tables only housed information on Fivetran Basic SQL Transformations, which were sunset last year ([PR #96](https://github.com/fivetran/dbt_fivetran_log/pull/96)). # dbt_fivetran_log v1.2.0 diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index b7702d6f..b2547c27 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,6 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 - +dbt-databricks>=1.6.0,<2.0.0 oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437 \ No newline at end of file diff --git a/integration_tests/seeds/.DS_Store b/integration_tests/seeds/.DS_Store index 638c07cccd4e0fa11d3753b97ae449228b495aab..effbdf8f05b6d85a1bc0e0dd9ebbf0efd8c4833f 100644 GIT binary patch delta 68 zcmZoMXfc=|#>B)qu~2NHo+2a1#(>?7j2xSJSoSk+4rJTLxUr#)X)`+qKL=3FWB!ku~2NHo+2aH#(>?7iv?Ji7`Zm{Fzsg)7Gfx2C}K!t$YUsGNMpzc zvU4X(Fso=Vocj+33?Lu~R+Pz*4uq*dCGkLhB0~~G4nyi>IVLG4hPurMm=7{;X6NAN d0NS;gk>xw{WPTAx4n_tBCXfM}BSh9P0|5F?B0~TG From 749145e0cfc64fc41f164d347c6e96936db28291 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:45:37 -0800 Subject: [PATCH 05/11] more stuff --- dbt_project.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 2fac2cab..b2e4dbf1 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -21,7 +21,5 @@ vars: destination: "{{ source('fivetran_platform', 'destination') }}" destination_membership: "{{ source('fivetran_platform', 'destination_membership') }}" log: "{{ source('fivetran_platform', 'log') }}" - transformation: "{{ source('fivetran_platform', 'transformation') }}" - trigger_table: "{{ source('fivetran_platform', 'trigger_table') }}" user: "{{ source('fivetran_platform', 'user') }}" usage_cost: "{{ source('fivetran_platform', 'usage_cost') }}" From 98c169e7d9e7c1801d997801652a2ba13f43f974 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:41:54 -0800 Subject: [PATCH 06/11] Readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d7923ecc..3ce0de26 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,6 @@ Refer to the table below for a detailed view of all models materialized by defau | [fivetran_platform__audit_table](https://fivetran.github.io/dbt_fivetran_log/#!/model/model.fivetran_log.fivetran_platform__audit_table) | Replaces the deprecated [`_fivetran_audit` table](https://fivetran.com/docs/getting-started/system-columns-and-tables#audittables). Each record represents a table being written to during a connector sync. Contains timestamps related to the connector and table-level sync progress and the sum of records inserted/replaced, updated, and deleted in the table. | - # 🎯 How do I use the dbt package? ## Step 1: Pre-Requisites - **Connector**: Have the Fivetran Platform connector syncing data into your warehouse. @@ -65,7 +64,7 @@ Include the following Fivetran Platform package version range in your `packages. ```yaml packages: - package: fivetran/fivetran_log - version: [">=1.2.0", "<1.3.0"] + version: [">=1.3.0", "<1.4.0"] ``` > Note that altough the source connector is now "Fivetran Platform", the package retains the old name of "fivetran_log". @@ -97,6 +96,7 @@ Some users may wish to exclude Fivetran error and warnings messages from the fin vars: fivetran_platform_using_sync_alert_messages: false # this will disable only the sync alert messages within the connector status model ``` + ### Change the Build Schema By default this package will build the Fivetran staging models within a schema titled ( + `_stg_fivetran_platform`) and the Fivetran Platform final models within your + `_fivetran_platform` in your target database. If this is not where you would like you Fivetran staging and final models to be written to, add the following configuration to your root `dbt_project.yml` file: @@ -107,7 +107,7 @@ models: staging: +schema: my_new_staging_models_schema # leave blank for just the target_schema ``` - + ### Change the Source Table References If an individual source table has a different name than expected (see this projects [dbt_project.yml](https://github.com/fivetran/dbt_fivetran_log/blob/main/dbt_project.yml) variable declarations for expected names), provide the name of the table as it appears in your warehouse to the respective variable as identified below: ```yml @@ -129,7 +129,7 @@ dispatch: Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Refer to the linked docs for more information on how to setup your project for orchestration through Fivetran.
- + # 🔍 Does this package have dependencies? This dbt package is dependent on the following dbt packages. Please be aware that these dependencies are installed by default within this package. For more information on the below packages, refer to the [dbt hub](https://hub.getdbt.com/) site. > **If you have any of these dependent packages in your own `packages.yml` I highly recommend you remove them to ensure there are no package version conflicts.** @@ -144,7 +144,7 @@ packages: - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] ``` - + # 🙌 How is this package maintained and can I contribute? ## Package Maintenance The Fivetran team maintaining this package **only** maintains the latest version of the package. We highly recommend you stay consistent with the [latest version](https://hub.getdbt.com/fivetran/fivetran_log/latest/) of the package and refer to the [CHANGELOG](https://github.com/fivetran/dbt_fivetran_log/blob/main/CHANGELOG.md) and release notes for more information on changes across versions. From 5d20cc4dc8ed6cb362d75ecc7a88599834657615 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:38:50 -0800 Subject: [PATCH 07/11] remove DS_STore --- integration_tests/.DS_Store | Bin 8196 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 integration_tests/.DS_Store diff --git a/integration_tests/.DS_Store b/integration_tests/.DS_Store deleted file mode 100644 index 8ea5ee8e1fac1657a3230f4daefd21dba416d54f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMF>ljA6n;)Zaa&L`AW@_udFfVFiJ+7PDGgyLQ>9ce0BY?dN&}7^5{FPiP)0;Y zHV}w^fM8%?LNLLOV5C0-fmFVCcXjMEk(f{`?n!s=`QH2P?tN$bat;w0`F1%=G)qJ# z3hVSVsu99|E(au`S{Q5qPmb$5u+u0j%BPiPKr^5j&Gy{i=0qof<=9qn7dUdTC z&ZxeOej#%42PHTD;$;nrF~CiBfmBvZ#Gj+^F`IdyXCR&mSk6azbGxAJ!L24Ah&!Iq=!b>DYG z+jg$iDxHn=sms0>*fp=x6yRF9f{>fnYhKy!_vo2q&`>WknvvIG{Tx=9VVw1Oe&L0_ zDDyM#-tIFW@ekM^<&Uqx51v4Xy;p2{uJ2XR#`B!U$~ah=4XTq5yG<@ts1;u)<18Yh z>ogOh8q@^tHZUdhvPj$L#~niN453pP!>1~>VjLv%zY!Puq8z;VyuP-(_mS_uh=0Jr zD1VHDkUVA(%XzN-1Y^bg(Y(KZ-AP=1ES|cdm_hF2Vq*Evy}~d`c-1Ww);Xp7Qgaix z>;0dJ(4rYQf(#_(oM-v`e`Wml|09^0p0Z{@Gw` Date: Thu, 9 Nov 2023 14:39:51 -0800 Subject: [PATCH 08/11] gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6b340720..0104f4b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ env/ target/ dbt_modules/ +dbt_packages/ +integration_tests/.DS_Store logs/ .gitignore .DS_Store From a46eb1006b94c1d84cd65fd782f6efa740c7be93 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:41:37 -0800 Subject: [PATCH 09/11] remove ds_store --- integration_tests/seeds/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 integration_tests/seeds/.DS_Store diff --git a/integration_tests/seeds/.DS_Store b/integration_tests/seeds/.DS_Store deleted file mode 100644 index effbdf8f05b6d85a1bc0e0dd9ebbf0efd8c4833f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOG*Pl5UolAMnW(v5f?Hi2zY}b#Dxo2t|gieDkK?6Ca&jV9?AoVzWNYa^8q&^ zS_M_Fr@LPF^y_ANnuxf!8lMw&h^Pq{tQ{h3F=-dyu$tFMBWoWkno~(#8qsniJ1Tyo z0smkSJR0ce_;)1MiclDae?)JphtT0y|AUP z+FR0BTSuQ`_e@YK@g|Es%$tPSLH+`)F-9vf>Hu?C>a*>_Z6mi6{t=iHFiP;}n2}(C z(oQzNB<#~2>~Ctpv!u@hn&J)a_q@SblHK(A?$L|7Nxm*!gis(9`1cCn%oc053?qgD zp+G3GRzSWF8C)=TtPJ(jfuL6aU<1`^D9g{Ha5Bf-u`)ypjExF3D*F|KjgI=v^K!?^ z(CEZ|`CuQJ{f5FiI{MEvoj5m)7z%^}pB1RvV@J;ai{1DC&!l(?1ww&8r2sej*Zm%* zj+G&5VDTfMGDHXkepG=Ezd(^d From a29680c8054da52d0cd28b8ff731ef2353231282 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:42:31 -0800 Subject: [PATCH 10/11] gitignore --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0104f4b2..4dab99d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ env/ target/ dbt_modules/ -dbt_packages/ -integration_tests/.DS_Store logs/ .gitignore .DS_Store -dbt_packages/ \ No newline at end of file +dbt_packages/ +integration_tests/.DS_Store +integration_tests/seeds/.DS_Store \ No newline at end of file From 616fcc6b5cbe1f413ebd97aef9fd2aad9d0b3f76 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 13 Nov 2023 13:14:56 -0800 Subject: [PATCH 11/11] joe feedback --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c4c132..010b5826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # dbt_fivetran_log v1.3.0 -## Breaking Changes +## 🚨 Breaking Changes 🚨 - Deprecated the `transformation` and `trigger_table` source tables and any downstream transforms. These tables only housed information on Fivetran Basic SQL Transformations, which were sunset last year ([PR #96](https://github.com/fivetran/dbt_fivetran_log/pull/96)). + - The entire `fivetran_platform__transformation_status` end model has therefore been removed. + - As they are now obsolete, the `fivetran_platform_using_transformations` and `fivetran_platform_using_triggers` variables have been removed. # dbt_fivetran_log v1.2.0