DS-3272 - Review checker data model for mobile (#4498)

* Add mobile shopping data * Remove the ff desktop from sql_generator * Fix build issue * Incorporate feedback from Bruce * Add clients table for mobile * FIX CI issue * Incorporate Bruce's feedback * Incorporate Curtis' feedback
mozilla · Nov 14, 2023 · abef158 · abef158
1 parent 77560c0
commit abef158
Show file tree

Hide file tree

Showing 31 changed files with 902 additions and 144 deletions.
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_clients/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_clients/view.sql
@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.review_checker_clients`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.review_checker_clients_v1`
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_events/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_events/view.sql
@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.review_checker_events`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.review_checker_events_v1`
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_microsurvey/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_microsurvey/view.sql
@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.review_checker_microsurvey`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.review_checker_microsurvey_v1`
diff --git a/...ates/review_checker_clients/metadata.yaml → ...d/review_checker_clients_v1/metadata.yaml b/...ates/review_checker_clients/metadata.yaml → ...d/review_checker_clients_v1/metadata.yaml
@@ -8,13 +8,13 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_review_checker
-  task_name: {{ app_name }}_review_checker_clients__v1
+  task_name: firefox_desktop_review_checker_clients__v1
 bigquery:
   time_partitioning:
     type: day
     field: submission_date
     require_partition_filter: true
   clustering:
     fields:
-      - normalized_channel
-      - sample_id
+    - normalized_channel
+    - sample_id
diff --git a/...emplates/review_checker_clients/query.sql → ...rived/review_checker_clients_v1/query.sql b/...emplates/review_checker_clients/query.sql → ...rived/review_checker_clients_v1/query.sql
@@ -25,7 +25,7 @@ WITH shopping_metrics AS (
     normalized_country_code,
     sample_id
   FROM
-    `{{ project_id }}.{{ app_name }}_stable.metrics_v1` AS m
+    `moz-fx-data-shared-prod.firefox_desktop_stable.metrics_v1`
   WHERE
     DATE(submission_timestamp) = @submission_date
   GROUP BY
@@ -116,4 +116,4 @@ joined_data AS (
 SELECT
   *
 FROM
-  joined_data;
+  joined_data
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/schema.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/schema.yaml
@@ -0,0 +1,40 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+- name: normalized_channel
+  type: STRING
+  mode: NULLABLE
+- name: normalized_country_code
+  type: STRING
+  mode: NULLABLE
+- name: legacy_client_id
+  type: STRING
+  mode: NULLABLE
+- name: client_id
+  type: STRING
+  mode: NULLABLE
+- name: sample_id
+  type: INTEGER
+  mode: NULLABLE
+- name: shopping_product_page_visits
+  type: INTEGER
+  mode: NULLABLE
+- name: is_opt_in
+  type: INTEGER
+  mode: NULLABLE
+- name: is_opt_out
+  type: INTEGER
+  mode: NULLABLE
+- name: sap
+  type: INTEGER
+  mode: NULLABLE
+- name: ad_click
+  type: INTEGER
+  mode: NULLABLE
+- name: active_hours_sum
+  type: FLOAT
+  mode: NULLABLE
+- name: is_fx_dau
+  type: INTEGER
+  mode: NULLABLE
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/metadata.yaml
@@ -0,0 +1,22 @@
+friendly_name: Review checker events data
+description: |-
+  The events table includes telemetry related to the review checker sidebar including user
+  interaction with the sidebar, open and closed states, and the status of reviews (stale, unavailable, etc.)
+owners:
+- [email protected]
+- [email protected]
+labels:
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_review_checker
+  task_name: firefox_desktop_review_checker_events__v1
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+  clustering:
+    fields:
+    - normalized_channel
+    - sample_id
diff --git a/...templates/review_checker_events/query.sql → ...erived/review_checker_events_v1/query.sql b/...templates/review_checker_events/query.sql → ...erived/review_checker_events_v1/query.sql
@@ -31,7 +31,7 @@ SELECT
   sample_id,
   mozfun.norm.truncate_version(client_info.app_display_version, "major") AS os_version,
 FROM
-  `{{ project_id }}.{{ app_name }}_stable.events_v1` AS e,
+  `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` AS e,
   UNNEST(events)
 WHERE
   DATE(submission_timestamp) = @submission_date

diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/schema.yaml
@@ -0,0 +1,55 @@
+fields:
+- name: submission_date
+  type: DATE
+  mode: NULLABLE
+- name: client_id
+  type: STRING
+  mode: NULLABLE
+- name: is_exposed_event
+  type: INTEGER
+  mode: NULLABLE
+- name: is_opt_in_event
+  type: INTEGER
+  mode: NULLABLE
+- name: is_surface_displayed
+  type: INTEGER
+  mode: NULLABLE
+- name: is_engaged_with_sidebar
+  type: INTEGER
+  mode: NULLABLE
+- name: experiments
+  type: RECORD
+  mode: REPEATED
+  fields:
+  - name: key
+    type: STRING
+    mode: NULLABLE
+  - name: value
+    type: RECORD
+    mode: NULLABLE
+    fields:
+    - name: branch
+      type: STRING
+      mode: NULLABLE
+    - name: extra
+      type: RECORD
+      mode: NULLABLE
+      fields:
+      - name: enrollment_id
+        type: STRING
+        mode: NULLABLE
+      - name: type
+        type: STRING
+        mode: NULLABLE
+- name: normalized_channel
+  type: STRING
+  mode: NULLABLE
+- name: normalized_country_code
+  type: STRING
+  mode: NULLABLE
+- name: sample_id
+  type: INTEGER
+  mode: NULLABLE
+- name: os_version
+  type: NUMERIC
+  mode: NULLABLE
diff --git a/.../review_checker_microsurvey/metadata.yaml → ...view_checker_microsurvey_v1/metadata.yaml b/.../review_checker_microsurvey/metadata.yaml → ...view_checker_microsurvey_v1/metadata.yaml
@@ -8,14 +8,13 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_review_checker
-  task_name: {{ app_name }}_review_checker_microsurvey__v1
+  task_name: firefox_desktop_review_checker_microsurvey__v1
 bigquery:
   time_partitioning:
     type: day
     field: submission_date
     require_partition_filter: true
   clustering:
     fields:
-      - normalized_channel
-      - sample_id
-
+    - normalized_channel
+    - sample_id
diff --git a/...ates/review_checker_microsurvey/query.sql → ...d/review_checker_microsurvey_v1/query.sql b/...ates/review_checker_microsurvey/query.sql → ...d/review_checker_microsurvey_v1/query.sql
@@ -68,7 +68,7 @@ filter_ms_microsurvey AS (
     END AS coded_answers,
     ping_info.experiments
   FROM
-    `{{ project_id }}.{{ app_name }}.messaging_system` msg
+    `moz-fx-data-shared-prod.firefox_desktop.messaging_system` msg
   LEFT JOIN
     `moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2` cfs
   ON

diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix/review_checker_events/view.sql b/sql/moz-fx-data-shared-prod/org_mozilla_fenix/review_checker_events/view.sql
@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.org_mozilla_fenix.review_checker_events`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.org_mozilla_fenix_derived.review_checker_events_v1`
diff --git a/...lates/review_checker_events/metadata.yaml → ...d/review_checker_clients_v1/metadata.yaml b/...lates/review_checker_events/metadata.yaml → ...d/review_checker_clients_v1/metadata.yaml
@@ -1,4 +1,4 @@
-friendly_name: Review checker events data
+friendly_name: Fenix review checker clients data
 description: |-
 owners:
 - [email protected]
@@ -8,13 +8,13 @@ labels:
   schedule: daily
 scheduling:
   dag_name: bqetl_review_checker
-  task_name: {{ app_name }}_review_checker_events__v1
+  task_name: org_mozilla_fenix_review_checker_clients__v1
 bigquery:
   time_partitioning:
     type: day
     field: submission_date
     require_partition_filter: true
   clustering:
     fields:
-      - normalized_channel
-      - sample_id
+    - normalized_channel
+    - sample_id
diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql
@@ -0,0 +1,107 @@
+WITH shopping_metrics AS (
+  SELECT
+    client_info.client_id AS client_id,
+    DATE(submission_timestamp) AS submission_date,
+    SUM(
+      CASE
+        WHEN metrics.counter.shopping_product_page_visits IS NOT NULL
+          THEN COALESCE(CAST(metrics.counter.shopping_product_page_visits AS INT64), 0)
+        ELSE 0
+      END
+    ) AS shopping_product_page_visits,
+    CASE
+      WHEN metrics.boolean.shopping_settings_user_has_onboarded = TRUE
+        AND metrics.boolean.shopping_settings_component_opted_out = FALSE
+        THEN 1
+      ELSE 0
+    END AS is_opt_in,
+    CASE
+      WHEN metrics.boolean.shopping_settings_component_opted_out = TRUE
+        THEN 1
+      ELSE 0
+    END AS is_opt_out,
+    CASE
+      WHEN metrics.boolean.shopping_settings_user_has_onboarded = TRUE
+        THEN 1
+      ELSE 0
+    END AS is_onboarded,
+    CASE
+      WHEN metrics.boolean.shopping_settings_nimbus_disabled_shopping = TRUE
+        THEN 1
+      ELSE 0
+    END AS is_nimbus_disabled,
+    normalized_channel,
+    normalized_country_code,
+    sample_id,
+    ANY_VALUE(ping_info.experiments) AS experiments,
+  FROM
+    `moz-fx-data-shared-prod.org_mozilla_fenix_stable.metrics_v1`
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+  GROUP BY
+    client_id,
+    submission_date,
+    is_opt_in,
+    is_opt_out,
+    normalized_channel,
+    normalized_country_code,
+    sample_id,
+    is_onboarded,
+    is_nimbus_disabled
+),
+active AS (
+  SELECT
+    client_info.client_id,
+    DATE(submission_timestamp) AS submission_date,
+    COALESCE(SUM(metrics.timespan.glean_baseline_duration.value), 0) / 3600.0 AS active_hours
+  FROM
+    `moz-fx-data-shared-prod.fenix.baseline`
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+  GROUP BY
+    client_info.client_id,
+    submission_date
+),
+search AS (
+  SELECT
+    submission_date,
+    client_id,
+    search_count AS sap,
+    ad_click
+  FROM
+    `moz-fx-data-shared-prod.search.mobile_search_clients_engines_sources_daily`
+  WHERE
+    submission_date = @submission_date
+    AND normalized_app_name = "Fenix"
+),
+joined_data AS (
+  SELECT
+    sm.submission_date,
+    normalized_channel,
+    normalized_country_code,
+    sm.client_id,
+    sm.sample_id,
+    sm.shopping_product_page_visits,
+    sm.experiments,
+    sm.is_opt_in,
+    sm.is_opt_out,
+    s.sap,
+    s.ad_click,
+    active.active_hours AS active_hours_sum
+  FROM
+    shopping_metrics sm
+  LEFT JOIN
+    active
+  ON
+    active.client_id = sm.client_id
+    AND active.submission_date = sm.submission_date
+  LEFT JOIN
+    search AS s
+  ON
+    s.client_id = sm.client_id
+    AND s.submission_date = sm.submission_date
+)
+SELECT
+  *
+FROM
+  joined_data