Merge pull request #49 from calogica/add/expect_grouped_row_values_to…

…_have_recent_data

Add test "expect_grouped_row_values_to_have_recent_data"

README.md

@@ -71,6 +71,31 @@ tests:
         interval: 1
 ```
 
+#### [expect_grouped_row_values_to_have_recent_data](macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql)
+
+Expect the model to have **grouped** rows that are at least as recent as the defined interval prior to the current timestamp.
+Uuse this to test whether there is recent data for each grouped row defined by `group_by` (which is a list of columns) and a `timestamp_column`.
+
+*Applies to:* Model, Seed, Source
+
+```yaml
+models: # or seeds:
+  - name : my_model
+    tests :
+        - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+            group_by: [group_id]
+            timestamp_column: date_day
+            datepart: day
+            interval: 1
+        # or also:
+        - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+            group_by: [group_id, other_group_id]
+            timestamp_column: date_day
+            datepart: day
+            interval: 1
+```
+
+
 #### [expect_table_column_count_to_be_between](macros/schema_tests/table_shape/expect_table_column_count_to_be_between.sql)
 
 Expect the number of columns in a model to be between two values.

integration_tests/models/schema_tests/schema.yml

@@ -254,3 +254,17 @@ models:
         - dbt_expectations.expect_table_row_count_to_equal_other_table_times_factor:
             compare_model: ref("data_test")
             factor: 2
+
+  - name : timeseries_data_grouped
+    tests :
+        - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+            group_by: [group_id]
+            timestamp_column: date_day
+            datepart: day
+            interval: 1
+        - dbt_expectations.expect_grouped_row_values_to_have_recent_data:
+            group_by: [group_id, row_value]
+            timestamp_column: date_day
+            datepart: day
+            interval: 1
+

integration_tests/models/schema_tests/timeseries_data_grouped.sql

@@ -0,0 +1,35 @@
+with dates as (
+    {{ dbt_date.get_base_dates(n_dateparts=12, datepart='month') }}
+),
+groupings as (
+    {{ dbt_utils.generate_series(upper_bound=4) }}
+),
+row_values as (
+    {{ dbt_utils.generate_series(upper_bound=10) }}
+),
+add_row_values as (
+
+    select
+        d.date_day,
+        cast(g.generated_number as {{ dbt_utils.type_int() }}) as group_id,
+        cast(floor(100 * r.generated_number) as {{ dbt_utils.type_int() }}) as row_value
+    from
+        dates d
+        cross join
+        groupings g
+        cross join
+        row_values r
+
+),
+add_logs as (
+
+    select
+        *,
+        {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log
+    from
+        add_row_values
+)
+select
+    *
+from
+    add_logs

macros/schema_tests/table_shape/expect_grouped_row_values_to_have_recent_data.sql

@@ -0,0 +1,24 @@
+{% macro test_expect_grouped_row_values_to_have_recent_data(model, group_by, timestamp_column, datepart, interval) %}
+with latest_grouped_timestamps as (
+
+    select
+        {%- for g in group_by %}
+        {{ g }},
+        {%- endfor %}
+        max({{ timestamp_column }}) as latest_timestamp_column
+    from
+        {{ model }}
+    {{ dbt_utils.group_by(group_by | length )}}
+
+),
+validation_errors as (
+
+    select *
+    from
+        latest_grouped_timestamps
+    where
+        latest_timestamp_column < {{ dbt_utils.dateadd(datepart, interval * -1, dbt_date.now()) }}
+
+)
+select count(*) from validation_errors
+{% endmacro %}