Merge pull request #79 from camcyr-at-brzwy/adds-sync-id-to-stg-logs

adds sync_id as well as docs
fivetran · Apr 4, 2023 · de9b667 · de9b667
2 parents f68d143 + 608797d
commit de9b667
Show file tree

Hide file tree

Showing 13 changed files with 85 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,18 @@
+# dbt_fivetran_log v0.7.4
+[PR #79](https://github.com/fivetran/dbt_fivetran_log/pull/79) includes the following updates:
+
+## Enhancements
+- The `sync_id` field from the source `log` table is added to the `stg_fivetran_log__log` model for ease of grouping events by the sync that they are associated with.
+
+## Under the Hood
+- Added the `get_log_columns` macro and included the fill staging cte's within the `stg_fivetran_log__log` model to ensure the model succeeds regardless of a user not having all the required fields.
+
+## Documentation Updates
+- The `sync_id` field is added to the documentation in the `fivetran_log.yml` file.
+
+## Contributors
+- [@camcyr-at-brzwy](https://github.com/camcyr-at-brzwy) ([#79](https://github.com/fivetran/dbt_fivetran_log/pull/79))
+
 # dbt_fivetran_log v0.7.3
 PR [#77](https://github.com/fivetran/dbt_fivetran_log/pull/77) includes the following updates:
 ## Bug Fixes

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '0.7.3'
+version: '0.7.4'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '0.7.3'
+version: '0.7.4'
 
 config-version: 2
 profile: 'integration_tests'

diff --git a/integration_tests/seeds/log.csv b/integration_tests/seeds/log.csv
@@ -1,25 +1,25 @@
-id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id
-legislative_lazy,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning,
-paint_wedges,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call,
-intrinsic_departed,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end,
-intrinsic_departed,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,
-intrinsic_departed,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,
-intrinsic_departed,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,
-intrinsic_departed,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start,
+id,time_stamp,_fivetran_synced,connector_id,event,message_data,message_event,transformation_id,sync_id
+legislative_lazy,2021-02-13 13:34:03.343,2021-02-16 21:11:07.065,legislative_lazy,WARNING,"{""type"":""table_excluded_by_system"",""message"":""salesforce.affectlayer__AffectLayer_User_Setting__ChangeEvent has been Excluded by system. Reason : Not queryable""}",warning,123
+paint_wedges,2021-02-12 08:15:05.555,2021-02-16 21:48:51.493,paint_wedges,INFO,"{""method"":""GET"",""uri"":""https://google.com""}",api_call,123
+intrinsic_departed,2021-12-09 14:27:00.504,2021-12-09 20:30:53.959,intrinsic_departed,INFO,,sync_end,123
+intrinsic_departed,2021-12-09 14:26:52.433,2021-12-09 20:30:53.957,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":4,""operationType"":""REPLACED_OR_INSERTED"",""table"":""fivetran_audit""}",records_modified,123
+intrinsic_departed,2021-12-09 14:26:52.242,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:51.703,2021-12-09 20:30:53.955,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:44,2021-12-09 20:30:53.952,intrinsic_departed,INFO,"{""table"":""fivetran_audit""}",write_to_table_start,123
+intrinsic_departed,2021-12-09 14:26:42.084,2021-12-09 20:30:53.950,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":9,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_history""}",records_modified,123
+intrinsic_departed,2021-12-09 14:26:42.029,2021-12-09 20:30:53.949,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":5,""operationType"":""REPLACED_OR_INSERTED"",""table"":""media_insights""}",records_modified,123
+intrinsic_departed,2021-12-09 14:26:41.886,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:41.875,2021-12-09 20:30:53.948,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:40.880,2021-12-09 20:30:53.946,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:40.737,2021-12-09 20:30:53.944,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:39.469,2021-12-09 20:30:53.935,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":1,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_history""}",records_modified,123
+intrinsic_departed,2021-12-09 14:26:39.446,2021-12-09 20:30:53.932,intrinsic_departed,INFO,"{""schema"":""instagram_business"",""count"":3,""operationType"":""REPLACED_OR_INSERTED"",""table"":""user_insights""}",records_modified,123
+intrinsic_departed,2021-12-09 14:26:39.092,2021-12-09 20:30:53.930,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:39.087,2021-12-09 20:30:53.929,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:37.735,2021-12-09 20:30:53.928,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:37.691,2021-12-09 20:30:53.920,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_end,123
+intrinsic_departed,2021-12-09 14:26:29.860,2021-12-09 20:30:53.904,intrinsic_departed,INFO,"{""table"":""media_history""}",write_to_table_start,123
+intrinsic_departed,2021-12-09 14:26:29.814,2021-12-09 20:30:53.903,intrinsic_departed,INFO,"{""table"":""user_insights""}",write_to_table_start,123
+intrinsic_departed,2021-12-09 14:26:29.744,2021-12-09 20:30:53.901,intrinsic_departed,INFO,"{""table"":""user_history""}",write_to_table_start,123
+intrinsic_departed,2021-12-09 14:26:29.719,2021-12-09 20:30:53.878,intrinsic_departed,INFO,"{""table"":""media_insights""}",write_to_table_start,123
+intrinsic_departed,2021-12-09 14:26:05.907,2021-12-09 20:30:53.778,intrinsic_departed,INFO,,sync_start,123
diff --git a/macros/get_log_columns.sql b/macros/get_log_columns.sql
@@ -0,0 +1,17 @@
+{% macro get_log_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "connector_id", "datatype": dbt.type_string()},
+    {"name": "event", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "message_data", "datatype": dbt.type_string()},
+    {"name": "message_event", "datatype": dbt.type_string()},
+    {"name": "sync_id", "datatype": dbt.type_string()},
+    {"name": "time_stamp", "datatype": dbt.type_timestamp()},
+    {"name": "transformation_id", "datatype": dbt.type_string()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}
diff --git a/models/staging/src_fivetran_log.yml b/models/staging/src_fivetran_log.yml
@@ -122,6 +122,8 @@ sources:
             description: The routine involved in the log, defined [here](https://fivetran.com/docs/logs#logevents).
           - name: transformation_id
             description: Foreign key referencing the `transformation` if the event refers to a transformation run.
+          - name: sync_id
+            description: Identifies which sync the event is associated with.
 
       - name: account
         identifier: "{{ var('fivetran_log_account_identifier', 'account')}}"

diff --git a/models/staging/stg_fivetran_log.yml b/models/staging/stg_fivetran_log.yml
@@ -115,6 +115,8 @@ models:
         description: >
           Contains either `connector_name` (for app/api logs), `transformation_id` (for transformation logs), 
           or the value `"system"` (for system generated logs).
+      - name: sync_id
+        description: Identifies which sync the event is associated with.
       - name: created_at
         description: Timestamp of when the event was logged.
       - name: connector_id

diff --git a/models/staging/stg_fivetran_log__log.sql b/models/staging/stg_fivetran_log__log.sql
@@ -1,13 +1,25 @@
-with log as (
+with base as (
 
     select * 
-    from {{ var('log') }}
+    from {{ ref('stg_fivetran_log__log_tmp') }}
 ),
 
 fields as (
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_fivetran_log__log_tmp')),
+                staging_columns=get_log_columns()
+            )
+        }}
+    from base
+),
+
+final as (
 
     select
         id as log_id, 
+        sync_id,
         cast(time_stamp as {{ dbt.type_timestamp() }}) as created_at,
         connector_id, -- Note: the connector_id column used to erroneously equal the connector_name, NOT its id.
         case when transformation_id is not null and event is null then 'TRANSFORMATION'
@@ -18,8 +30,8 @@ fields as (
         when transformation_id is not null and message_data like '%has failed%' then 'transformation run failed'
         else message_event end as event_subtype,
         transformation_id
-    from log
+    from fields
 )
 
 select * 
-from fields 
+from final 
diff --git a/models/staging/tmp/stg_fivetran_log__log_tmp.sql b/models/staging/tmp/stg_fivetran_log__log_tmp.sql
@@ -0,0 +1,2 @@
+select *
+from {{ var('log') }}