DENG-476 - Update monitoring ETLs to reference main_v5 (#4431)

mozilla · Nov 14, 2023 · 42ab510 · 42ab510
1 parent 659dd58
commit 42ab510
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 11 deletions.
diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/column_size_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/column_size_v1/metadata.yaml
@@ -9,7 +9,7 @@ scheduling:
   dag_name: bqetl_monitoring
   arguments: ["--date", "{{ ds }}"]
   referenced_tables:
-    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
+    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v5']
 bigquery:
   time_partitioning:
     type: day

diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql b/sql/moz-fx-data-shared-prod/monitoring_derived/shredder_progress/view.sql
@@ -5,10 +5,10 @@ WITH shredder AS (
   SELECT
     task_id,
     CASE
-      WHEN target = "moz-fx-data-shared-prod.telemetry_stable.main_v4"
-        THEN "telemetry_main"
-      WHEN target = "moz-fx-data-shared-prod.telemetry_derived.main_summary_v4"
-        THEN "telemetry_main_summary"
+      WHEN target = "moz-fx-data-shared-prod.telemetry_stable.main_v5"
+        THEN "telemetry_main_v5"
+      WHEN target = "moz-fx-data-shared-prod.telemetry_stable.main_use_counter_v4"
+        THEN "telemetry_main_use_counter"
       ELSE "all"
     END AS airflow_task_id,
     target,

diff --git a/...oz-fx-data-shared-prod/monitoring_derived/stable_and_derived_table_sizes_v1/metadata.yaml b/...oz-fx-data-shared-prod/monitoring_derived/stable_and_derived_table_sizes_v1/metadata.yaml
@@ -13,7 +13,7 @@ scheduling:
   arguments: ["--date", "{{ macros.ds_add(ds, -1) }}"]
   referenced_tables:
     - ['moz-fx-data-shared-prod', '*_stable', '*']
-    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
+    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v5']
     - ['moz-fx-data-shared-prod', '*_derived', '*']
 bigquery:
   time_partitioning:

diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/stable_table_column_counts_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/stable_table_column_counts_v1/metadata.yaml
@@ -10,7 +10,7 @@ scheduling:
   dag_name: bqetl_monitoring
   referenced_tables:
   - ['moz-fx-data-shared-prod', '*_stable', '*']
-  - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
+  - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v5']
   date_partition_parameter: null
   parameters: ["submission_date:DATE:{{ds}}"]
 bigquery:

diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/structured_distinct_docids_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/structured_distinct_docids_v1/metadata.yaml
@@ -24,4 +24,4 @@ scheduling:
        'monitoring',
        'payload_bytes_decoded_stub_installer']
     - ['moz-fx-data-shared-prod', '*_stable', '*']
-    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
+    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v5']
diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/structured_missing_columns_v1/query.py b/sql/moz-fx-data-shared-prod/monitoring_derived/structured_missing_columns_v1/query.py
@@ -36,7 +36,7 @@
     DATE(submission_timestamp) = date "{date}"
     -- https://cloud.google.com/bigquery/docs/querying-wildcard-tables#filtering_selected_tables_using_table_suffix
     -- IT's also possible to exclude tables in this query e.g
-    -- AND _TABLE_SUFFIX NOT IN ('main_v4', 'saved_session_v4', 'first_shutdown_v4')
+    -- AND _TABLE_SUFFIX NOT IN ('main_v5', 'saved_session_v5', 'first_shutdown_v5')
 ),
 transformed AS (
   SELECT

diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_distinct_docids_v1/metadata.yaml
@@ -20,5 +20,5 @@ scheduling:
        'monitoring',
        'payload_bytes_decoded_telemetry']
     - ['moz-fx-data-shared-prod', 'telemetry_stable', '*']
-    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
+    - ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v5']
     - ['moz-fx-data-shared-prod', 'telemetry_live', '*']
diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v2/view.sql b/sql/moz-fx-data-shared-prod/monitoring_derived/telemetry_missing_columns_v2/view.sql
@@ -26,7 +26,14 @@ extracted AS (
     )
     -- https://cloud.google.com/bigquery/docs/querying-wildcard-tables#filtering_selected_tables_using_table_suffix
     -- exclude pings derived from main schema to save on space, 300GB vs 3TB
-    AND _TABLE_SUFFIX NOT IN ('main_v4', 'saved_session_v4', 'first_shutdown_v4')
+    AND _TABLE_SUFFIX NOT IN (
+      'main_v4',
+      'saved_session_v4',
+      'first_shutdown_v4',
+      'main_v5',
+      'saved_session_v5',
+      'first_shutdown_v5'
+    )
     AND _TABLE_SUFFIX NOT IN (SELECT * FROM placeholder_table_names)
 ),
 transformed AS (