Skip to content

Commit

Permalink
Apply shredder_mitigation label and +1 day offset in scheduling. (#6792)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucia-vargas-a
lucia-vargas-a authored Jan 10, 2025
1 parent fd2f3a2 commit 3fcd587
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 0 deletions.
2 changes: 2 additions & 0 deletions sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ owners:
- [email protected]
labels:
incremental: true
shredder_mitigation: true
scheduling:
dag_name: bqetl_analytics_aggregations
date_partition_offset: -1
bigquery:
time_partitioning:
type: day
Expand Down
2 changes: 2 additions & 0 deletions sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ owners:
- [email protected]
labels:
incremental: true
shredder_mitigation: true
scheduling:
dag_name: bqetl_analytics_aggregations
date_partition_offset: -1
bigquery:
time_partitioning:
type: day
Expand Down

1 comment on commit 3fcd587

@dataops-ci-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Integration report for "Apply shredder_mitigation label and +1 day offset in scheduling. (#6792)"

sql.diff

Click to expand! 
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py
--- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py	2025-01-10 18:26:05.000000000 +0000
+++ /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py	2025-01-10 18:28:34.000000000 +0000
@@ -834,7 +834,7 @@
 
     fenix_derived__locale_aggregates__v1 = bigquery_etl_query(
         task_id="fenix_derived__locale_aggregates__v1",
-        destination_table="locale_aggregates_v1",
+        destination_table='locale_aggregates_v1${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}',
         dataset_id="fenix_derived",
         project_id="moz-fx-data-shared-prod",
         owner="[email protected]",
@@ -843,8 +843,9 @@
             "[email protected]",
             "[email protected]",
         ],
-        date_partition_parameter="submission_date",
+        date_partition_parameter=None,
         depends_on_past=False,
+        parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"],
     )
 
     firefox_desktop_active_users_aggregates_v3 = bigquery_etl_query(
@@ -902,7 +903,7 @@
 
     firefox_desktop_derived__locale_aggregates__v1 = bigquery_etl_query(
         task_id="firefox_desktop_derived__locale_aggregates__v1",
-        destination_table="locale_aggregates_v1",
+        destination_table='locale_aggregates_v1${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}',
         dataset_id="firefox_desktop_derived",
         project_id="moz-fx-data-shared-prod",
         owner="[email protected]",
@@ -911,8 +912,9 @@
             "[email protected]",
             "[email protected]",
         ],
-        date_partition_parameter="submission_date",
+        date_partition_parameter=None,
         depends_on_past=False,
+        parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"],
     )
 
     firefox_ios_active_users_aggregates_v3 = bigquery_etl_query(
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/metadata.yaml	2025-01-10 18:22:30.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/locale_aggregates_v1/metadata.yaml	2025-01-10 18:22:57.000000000 +0000
@@ -5,10 +5,12 @@
 - [email protected]
 labels:
   incremental: true
+  shredder_mitigation: true
   dag: bqetl_analytics_aggregations
   owner1: lvargas
 scheduling:
   dag_name: bqetl_analytics_aggregations
+  date_partition_offset: -1
 bigquery:
   time_partitioning:
     type: day
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/metadata.yaml	2025-01-10 18:22:24.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/locale_aggregates_v1/metadata.yaml	2025-01-10 18:23:01.000000000 +0000
@@ -5,10 +5,12 @@
 - [email protected]
 labels:
   incremental: true
+  shredder_mitigation: true
   dag: bqetl_analytics_aggregations
   owner1: lvargas
 scheduling:
   dag_name: bqetl_analytics_aggregations
+  date_partition_offset: -1
 bigquery:
   time_partitioning:
     type: day

Link to full diff

Please sign in to comment.