From abef158bc2783ca3a3243958d0eb0756a14a913a Mon Sep 17 00:00:00 2001 From: Alekhya <88394696+alekhyamoz@users.noreply.github.com> Date: Thu, 9 Nov 2023 08:54:18 -0500 Subject: [PATCH] DS-3272 - Review checker data model for mobile (#4498) * Add mobile shopping data * Remove the ff desktop from sql_generator * Fix build issue * Incorporate feedback from Bruce * Add clients table for mobile * FIX CI issue * Incorporate Bruce's feedback * Incorporate Curtis' feedback --- .../review_checker_clients/view.sql | 7 ++ .../review_checker_events/view.sql | 7 ++ .../review_checker_microsurvey/view.sql | 7 ++ .../review_checker_clients_v1}/metadata.yaml | 6 +- .../review_checker_clients_v1}/query.sql | 4 +- .../review_checker_clients_v1/schema.yaml | 40 +++++++ .../review_checker_events_v1/metadata.yaml | 22 ++++ .../review_checker_events_v1}/query.sql | 2 +- .../review_checker_events_v1/schema.yaml | 55 +++++++++ .../metadata.yaml | 7 +- .../review_checker_microsurvey_v1}/query.sql | 2 +- .../review_checker_events/view.sql | 7 ++ .../review_checker_clients_v1}/metadata.yaml | 8 +- .../review_checker_clients_v1/query.sql | 107 +++++++++++++++++ .../review_checker_clients_v1/schema.yaml | 58 ++++++++++ .../review_checker_events_v1/metadata.yaml | 22 ++++ .../review_checker_events_v1/query.sql | 60 ++++++++++ .../review_checker_events_v1/schema.yaml | 103 +++++++++++++++++ .../review_checker_clients/view.sql | 7 ++ .../review_checker_events/view.sql | 7 ++ .../review_checker_clients_v1/metadata.yaml | 20 ++++ .../review_checker_clients_v1/query.sql | 108 +++++++++++++++++ .../review_checker_clients_v1/schema.yaml | 58 ++++++++++ .../review_checker_events_v1/metadata.yaml | 22 ++++ .../review_checker_events_v1/query.sql | 62 ++++++++++ .../review_checker_events_v1/schema.yaml | 109 ++++++++++++++++++ sql_generators/review_checker/__init__.py | 96 --------------- .../templates/review_checker_clients/view.sql | 8 -- .../templates/review_checker_events/view.sql | 8 -- .../review_checker_microsurvey/view.sql | 8 -- .../review_checker/templates/templating.yaml | 9 -- 31 files changed, 902 insertions(+), 144 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_clients/view.sql create mode 100644 sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_events/view.sql create mode 100644 sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_microsurvey/view.sql rename {sql_generators/review_checker/templates/review_checker_clients => sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1}/metadata.yaml (77%) rename {sql_generators/review_checker/templates/review_checker_clients => sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1}/query.sql (97%) create mode 100644 sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/metadata.yaml rename {sql_generators/review_checker/templates/review_checker_events => sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1}/query.sql (95%) create mode 100644 sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/schema.yaml rename {sql_generators/review_checker/templates/review_checker_microsurvey => sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1}/metadata.yaml (76%) rename {sql_generators/review_checker/templates/review_checker_microsurvey => sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1}/query.sql (98%) create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_fenix/review_checker_events/view.sql rename {sql_generators/review_checker/templates/review_checker_events => sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1}/metadata.yaml (66%) create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_clients/view.sql create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_events/view.sql create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/schema.yaml delete mode 100644 sql_generators/review_checker/__init__.py delete mode 100644 sql_generators/review_checker/templates/review_checker_clients/view.sql delete mode 100644 sql_generators/review_checker/templates/review_checker_events/view.sql delete mode 100644 sql_generators/review_checker/templates/review_checker_microsurvey/view.sql delete mode 100644 sql_generators/review_checker/templates/templating.yaml diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_clients/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_clients/view.sql new file mode 100644 index 00000000000..7918d77c10e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_clients/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.firefox_desktop.review_checker_clients` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.firefox_desktop_derived.review_checker_clients_v1` diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_events/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_events/view.sql new file mode 100644 index 00000000000..a83377dc9d2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_events/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.firefox_desktop.review_checker_events` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.firefox_desktop_derived.review_checker_events_v1` diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_microsurvey/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_microsurvey/view.sql new file mode 100644 index 00000000000..1fe90d97936 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_desktop/review_checker_microsurvey/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.firefox_desktop.review_checker_microsurvey` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.firefox_desktop_derived.review_checker_microsurvey_v1` diff --git a/sql_generators/review_checker/templates/review_checker_clients/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/metadata.yaml similarity index 77% rename from sql_generators/review_checker/templates/review_checker_clients/metadata.yaml rename to sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/metadata.yaml index 29c1c527656..e59158c0b4e 100644 --- a/sql_generators/review_checker/templates/review_checker_clients/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/metadata.yaml @@ -8,7 +8,7 @@ labels: schedule: daily scheduling: dag_name: bqetl_review_checker - task_name: {{ app_name }}_review_checker_clients__v1 + task_name: firefox_desktop_review_checker_clients__v1 bigquery: time_partitioning: type: day @@ -16,5 +16,5 @@ bigquery: require_partition_filter: true clustering: fields: - - normalized_channel - - sample_id + - normalized_channel + - sample_id diff --git a/sql_generators/review_checker/templates/review_checker_clients/query.sql b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/query.sql similarity index 97% rename from sql_generators/review_checker/templates/review_checker_clients/query.sql rename to sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/query.sql index ab4c4be9297..5e55af65c52 100644 --- a/sql_generators/review_checker/templates/review_checker_clients/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/query.sql @@ -25,7 +25,7 @@ WITH shopping_metrics AS ( normalized_country_code, sample_id FROM - `{{ project_id }}.{{ app_name }}_stable.metrics_v1` AS m + `moz-fx-data-shared-prod.firefox_desktop_stable.metrics_v1` WHERE DATE(submission_timestamp) = @submission_date GROUP BY @@ -116,4 +116,4 @@ joined_data AS ( SELECT * FROM - joined_data; + joined_data diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/schema.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/schema.yaml new file mode 100644 index 00000000000..5c120fa4f42 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_clients_v1/schema.yaml @@ -0,0 +1,40 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: normalized_country_code + type: STRING + mode: NULLABLE +- name: legacy_client_id + type: STRING + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: shopping_product_page_visits + type: INTEGER + mode: NULLABLE +- name: is_opt_in + type: INTEGER + mode: NULLABLE +- name: is_opt_out + type: INTEGER + mode: NULLABLE +- name: sap + type: INTEGER + mode: NULLABLE +- name: ad_click + type: INTEGER + mode: NULLABLE +- name: active_hours_sum + type: FLOAT + mode: NULLABLE +- name: is_fx_dau + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/metadata.yaml new file mode 100644 index 00000000000..4fc82104370 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Review checker events data +description: |- + The events table includes telemetry related to the review checker sidebar including user + interaction with the sidebar, open and closed states, and the status of reviews (stale, unavailable, etc.) +owners: +- akommasani@mozilla.com +- betling@mozilla.com +labels: + incremental: true + schedule: daily +scheduling: + dag_name: bqetl_review_checker + task_name: firefox_desktop_review_checker_events__v1 +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + clustering: + fields: + - normalized_channel + - sample_id diff --git a/sql_generators/review_checker/templates/review_checker_events/query.sql b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/query.sql similarity index 95% rename from sql_generators/review_checker/templates/review_checker_events/query.sql rename to sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/query.sql index 15751ddaaed..5525eae9337 100644 --- a/sql_generators/review_checker/templates/review_checker_events/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/query.sql @@ -31,7 +31,7 @@ SELECT sample_id, mozfun.norm.truncate_version(client_info.app_display_version, "major") AS os_version, FROM - `{{ project_id }}.{{ app_name }}_stable.events_v1` AS e, + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` AS e, UNNEST(events) WHERE DATE(submission_timestamp) = @submission_date diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/schema.yaml new file mode 100644 index 00000000000..b8386d7d5e0 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_events_v1/schema.yaml @@ -0,0 +1,55 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: is_exposed_event + type: INTEGER + mode: NULLABLE +- name: is_opt_in_event + type: INTEGER + mode: NULLABLE +- name: is_surface_displayed + type: INTEGER + mode: NULLABLE +- name: is_engaged_with_sidebar + type: INTEGER + mode: NULLABLE +- name: experiments + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: RECORD + mode: NULLABLE + fields: + - name: branch + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: NULLABLE + fields: + - name: enrollment_id + type: STRING + mode: NULLABLE + - name: type + type: STRING + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: normalized_country_code + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: os_version + type: NUMERIC + mode: NULLABLE diff --git a/sql_generators/review_checker/templates/review_checker_microsurvey/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1/metadata.yaml similarity index 76% rename from sql_generators/review_checker/templates/review_checker_microsurvey/metadata.yaml rename to sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1/metadata.yaml index 5c06f8ff802..0005cc4d762 100644 --- a/sql_generators/review_checker/templates/review_checker_microsurvey/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1/metadata.yaml @@ -8,7 +8,7 @@ labels: schedule: daily scheduling: dag_name: bqetl_review_checker - task_name: {{ app_name }}_review_checker_microsurvey__v1 + task_name: firefox_desktop_review_checker_microsurvey__v1 bigquery: time_partitioning: type: day @@ -16,6 +16,5 @@ bigquery: require_partition_filter: true clustering: fields: - - normalized_channel - - sample_id - + - normalized_channel + - sample_id diff --git a/sql_generators/review_checker/templates/review_checker_microsurvey/query.sql b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1/query.sql similarity index 98% rename from sql_generators/review_checker/templates/review_checker_microsurvey/query.sql rename to sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1/query.sql index feba837d7c0..f02f9ed66fd 100644 --- a/sql_generators/review_checker/templates/review_checker_microsurvey/query.sql +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/review_checker_microsurvey_v1/query.sql @@ -68,7 +68,7 @@ filter_ms_microsurvey AS ( END AS coded_answers, ping_info.experiments FROM - `{{ project_id }}.{{ app_name }}.messaging_system` msg + `moz-fx-data-shared-prod.firefox_desktop.messaging_system` msg LEFT JOIN `moz-fx-data-shared-prod.telemetry_derived.clients_first_seen_v2` cfs ON diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix/review_checker_events/view.sql b/sql/moz-fx-data-shared-prod/org_mozilla_fenix/review_checker_events/view.sql new file mode 100644 index 00000000000..2c3f8988868 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix/review_checker_events/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.org_mozilla_fenix.review_checker_events` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.org_mozilla_fenix_derived.review_checker_events_v1` diff --git a/sql_generators/review_checker/templates/review_checker_events/metadata.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/metadata.yaml similarity index 66% rename from sql_generators/review_checker/templates/review_checker_events/metadata.yaml rename to sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/metadata.yaml index 184c23b35f9..e694236272f 100644 --- a/sql_generators/review_checker/templates/review_checker_events/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/metadata.yaml @@ -1,4 +1,4 @@ -friendly_name: Review checker events data +friendly_name: Fenix review checker clients data description: |- owners: - akommasani@mozilla.com @@ -8,7 +8,7 @@ labels: schedule: daily scheduling: dag_name: bqetl_review_checker - task_name: {{ app_name }}_review_checker_events__v1 + task_name: org_mozilla_fenix_review_checker_clients__v1 bigquery: time_partitioning: type: day @@ -16,5 +16,5 @@ bigquery: require_partition_filter: true clustering: fields: - - normalized_channel - - sample_id + - normalized_channel + - sample_id diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql new file mode 100644 index 00000000000..6479f75bd6b --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/query.sql @@ -0,0 +1,107 @@ +WITH shopping_metrics AS ( + SELECT + client_info.client_id AS client_id, + DATE(submission_timestamp) AS submission_date, + SUM( + CASE + WHEN metrics.counter.shopping_product_page_visits IS NOT NULL + THEN COALESCE(CAST(metrics.counter.shopping_product_page_visits AS INT64), 0) + ELSE 0 + END + ) AS shopping_product_page_visits, + CASE + WHEN metrics.boolean.shopping_settings_user_has_onboarded = TRUE + AND metrics.boolean.shopping_settings_component_opted_out = FALSE + THEN 1 + ELSE 0 + END AS is_opt_in, + CASE + WHEN metrics.boolean.shopping_settings_component_opted_out = TRUE + THEN 1 + ELSE 0 + END AS is_opt_out, + CASE + WHEN metrics.boolean.shopping_settings_user_has_onboarded = TRUE + THEN 1 + ELSE 0 + END AS is_onboarded, + CASE + WHEN metrics.boolean.shopping_settings_nimbus_disabled_shopping = TRUE + THEN 1 + ELSE 0 + END AS is_nimbus_disabled, + normalized_channel, + normalized_country_code, + sample_id, + ANY_VALUE(ping_info.experiments) AS experiments, + FROM + `moz-fx-data-shared-prod.org_mozilla_fenix_stable.metrics_v1` + WHERE + DATE(submission_timestamp) = @submission_date + GROUP BY + client_id, + submission_date, + is_opt_in, + is_opt_out, + normalized_channel, + normalized_country_code, + sample_id, + is_onboarded, + is_nimbus_disabled +), +active AS ( + SELECT + client_info.client_id, + DATE(submission_timestamp) AS submission_date, + COALESCE(SUM(metrics.timespan.glean_baseline_duration.value), 0) / 3600.0 AS active_hours + FROM + `moz-fx-data-shared-prod.fenix.baseline` + WHERE + DATE(submission_timestamp) = @submission_date + GROUP BY + client_info.client_id, + submission_date +), +search AS ( + SELECT + submission_date, + client_id, + search_count AS sap, + ad_click + FROM + `moz-fx-data-shared-prod.search.mobile_search_clients_engines_sources_daily` + WHERE + submission_date = @submission_date + AND normalized_app_name = "Fenix" +), +joined_data AS ( + SELECT + sm.submission_date, + normalized_channel, + normalized_country_code, + sm.client_id, + sm.sample_id, + sm.shopping_product_page_visits, + sm.experiments, + sm.is_opt_in, + sm.is_opt_out, + s.sap, + s.ad_click, + active.active_hours AS active_hours_sum + FROM + shopping_metrics sm + LEFT JOIN + active + ON + active.client_id = sm.client_id + AND active.submission_date = sm.submission_date + LEFT JOIN + search AS s + ON + s.client_id = sm.client_id + AND s.submission_date = sm.submission_date +) +SELECT + * +FROM + joined_data diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/schema.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/schema.yaml new file mode 100644 index 00000000000..6a3eecc656a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_clients_v1/schema.yaml @@ -0,0 +1,58 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: normalized_country_code + type: STRING + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: shopping_product_page_visits + type: INTEGER + mode: NULLABLE +- name: experiments + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: RECORD + mode: NULLABLE + fields: + - name: branch + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: NULLABLE + fields: + - name: enrollment_id + type: STRING + mode: NULLABLE + - name: type + type: STRING + mode: NULLABLE +- name: is_opt_in + type: INTEGER + mode: NULLABLE +- name: is_opt_out + type: INTEGER + mode: NULLABLE +- name: sap + type: INTEGER + mode: NULLABLE +- name: ad_click + type: INTEGER + mode: NULLABLE +- name: active_hours_sum + type: FLOAT + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/metadata.yaml new file mode 100644 index 00000000000..48ba85afb7f --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Fenix review checker events data +description: |- + The events table includes telemetry related to the review checker sidebar including user + interaction with the sidebar, open and closed states, and the status of reviews (stale, unavailable, etc.) +owners: +- akommasani@mozilla.com +- betling@mozilla.com +labels: + incremental: true + schedule: daily +scheduling: + dag_name: bqetl_review_checker + task_name: org_mozilla_fenix_review_checker_events__v1 +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + clustering: + fields: + - normalized_channel + - sample_id diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/query.sql b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/query.sql new file mode 100644 index 00000000000..b303939fed8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/query.sql @@ -0,0 +1,60 @@ +SELECT + DATE(submission_timestamp) AS submission_date, + client_info.client_id AS client_id, + client_info.os AS os, + client_info.os_version AS os_version, + MAX(CASE WHEN name = 'address_bar_feature_callout_displayed' THEN 1 ELSE 0 END) is_address_bar_feature_callout_displayed, + MAX(CASE WHEN name = 'address_bar_icon_clicked' THEN 1 ELSE 0 END) is_address_bar_icon_clicked, + MAX(CASE WHEN name = 'address_bar_icon_displayed' THEN 1 ELSE 0 END) is_address_bar_icon_displayed, + MAX(CASE WHEN name = 'surface_analyze_reviews_none_available_clicked' THEN 1 ELSE 0 END) is_surface_analyze_reviews_none_available_clicked, + MAX(CASE WHEN name = 'surface_closed' THEN 1 ELSE 0 END) is_surface_closed, + MAX(CASE WHEN name = 'surface_displayed' THEN 1 ELSE 0 END) is_surface_displayed, + MAX(CASE WHEN name = 'surface_expand_settings' THEN 1 ELSE 0 END) is_surface_expand_settings, + MAX(CASE WHEN name = 'surface_learn_more_clicked' THEN 1 ELSE 0 END) is_surface_learn_more_clicked, + MAX(CASE WHEN name = 'surface_no_review_reliability_available' THEN 1 ELSE 0 END) is_surface_no_review_reliability_available, + MAX(CASE WHEN name = 'surface_onboarding_displayed' THEN 1 ELSE 0 END) is_surface_onboarding_displayed, + MAX(CASE WHEN name = 'surface_opt_in_accepted' THEN 1 ELSE 0 END) surface_opt_in_accepted, + MAX(CASE WHEN name = 'surface_reactivated_button_clicked' THEN 1 ELSE 0 END) is_surface_reactivated_button_clicked, + MAX(CASE WHEN name = 'surface_reanalyze_clicked' THEN 1 ELSE 0 END) is_surface_reanalyze_clicked, + MAX(CASE WHEN name = 'surface_show_more_recent_reviews_clicked' THEN 1 ELSE 0 END) is_surface_show_more_recent_reviews_clicked, + MAX(CASE WHEN name = 'surface_show_privacy_policy_clicked' THEN 1 ELSE 0 END) is_surface_show_privacy_policy_clicked, + MAX(CASE WHEN name = 'surface_review_quality_explainer_url_clicked' THEN 1 ELSE 0 END) is_surface_show_quality_explainer_url_clicked, + MAX(CASE WHEN name = 'surface_show_terms_clicked' THEN 1 ELSE 0 END) is_surface_show_terms_clicked, + MAX( + CASE + WHEN name IN ( + 'surface_analyze_reviews_none_available_clicked', + 'surface_learn_more_clicked', + 'surface_no_review_reliability_available', + 'surface_reactivated_button_clicked', + 'surface_reanalyze_clicked', + 'surface_expand_settings', + 'surface_show_more_recent_reviews_clicked', + 'surface_show_privacy_policy_clicked', + 'surface_review_quality_explainer_url_clicked', + 'surface_show_terms_clicked' + ) + THEN 1 + ELSE 0 + END + ) is_engaged_with_sidebar, + ANY_VALUE(ping_info.experiments) AS experiments, + normalized_channel, + normalized_country_code, + sample_id, + mozfun.norm.truncate_version(client_info.app_display_version, "major") AS app_version, +FROM + `moz-fx-data-shared-prod.org_mozilla_fenix_stable.events_v1` AS e, + UNNEST(events) +WHERE + DATE(submission_timestamp) = @submission_date + AND category = 'shopping' +GROUP BY + DATE(submission_timestamp), + client_info.client_id, + normalized_channel, + normalized_country_code, + sample_id, + os, + os_version, + app_version diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/schema.yaml new file mode 100644 index 00000000000..60756e5bf0e --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_fenix_derived/review_checker_events_v1/schema.yaml @@ -0,0 +1,103 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: os + type: STRING + mode: NULLABLE +- name: os_version + type: STRING + mode: NULLABLE +- name: is_address_bar_feature_callout_displayed + type: INTEGER + mode: NULLABLE +- name: is_address_bar_icon_clicked + type: INTEGER + mode: NULLABLE +- name: is_address_bar_icon_displayed + type: INTEGER + mode: NULLABLE +- name: is_surface_analyze_reviews_none_available_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_closed + type: INTEGER + mode: NULLABLE +- name: is_surface_displayed + type: INTEGER + mode: NULLABLE +- name: is_surface_expand_settings + type: INTEGER + mode: NULLABLE +- name: is_surface_learn_more_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_no_review_reliability_available + type: INTEGER + mode: NULLABLE +- name: is_surface_onboarding_displayed + type: INTEGER + mode: NULLABLE +- name: surface_opt_in_accepted + type: INTEGER + mode: NULLABLE +- name: is_surface_reactivated_button_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_reanalyze_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_more_recent_reviews_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_privacy_policy_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_quality_explainer_url_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_terms_clicked + type: INTEGER + mode: NULLABLE +- name: is_engaged_with_sidebar + type: INTEGER + mode: NULLABLE +- name: experiments + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: RECORD + mode: NULLABLE + fields: + - name: branch + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: NULLABLE + fields: + - name: enrollment_id + type: STRING + mode: NULLABLE + - name: type + type: STRING + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: normalized_country_code + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: app_version + type: NUMERIC + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_clients/view.sql b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_clients/view.sql new file mode 100644 index 00000000000..32022c3de49 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_clients/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.org_mozilla_ios_firefox.review_checker_clients` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.review_checker_clients_v1` diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_events/view.sql b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_events/view.sql new file mode 100644 index 00000000000..b4491eaa1c4 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox/review_checker_events/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.org_mozilla_ios_firefox.review_checker_events` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.review_checker_events_v1` diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/metadata.yaml new file mode 100644 index 00000000000..6ecd73ab430 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/metadata.yaml @@ -0,0 +1,20 @@ +friendly_name: Firefox iOS review checker clients data +description: |- +owners: +- akommasani@mozilla.com +- betling@mozilla.com +labels: + incremental: true + schedule: daily +scheduling: + dag_name: bqetl_review_checker + task_name: org_mozilla_ios_firefox_review_checker_clients__v1 +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + clustering: + fields: + - normalized_channel + - sample_id diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/query.sql b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/query.sql new file mode 100644 index 00000000000..40d19779772 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/query.sql @@ -0,0 +1,108 @@ +WITH shopping_metrics AS ( + SELECT + client_info.client_id AS client_id, + DATE(submission_timestamp) AS submission_date, + SUM( + CASE + WHEN metrics.counter.shopping_product_page_visits IS NOT NULL + THEN COALESCE(CAST(metrics.counter.shopping_product_page_visits AS INT64), 0) + ELSE 0 + END + ) AS shopping_product_page_visits, + CASE + WHEN metrics.boolean.shopping_settings_user_has_onboarded = TRUE + AND metrics.boolean.shopping_settings_component_opted_out = FALSE + THEN 1 + ELSE 0 + END AS is_opt_in, + CASE + WHEN metrics.boolean.shopping_settings_component_opted_out = TRUE + THEN 1 + ELSE 0 + END AS is_opt_out, + CASE + WHEN metrics.boolean.shopping_settings_user_has_onboarded = TRUE + THEN 1 + ELSE 0 + END AS is_onboarded, + CASE + WHEN metrics.boolean.shopping_settings_nimbus_disabled_shopping = TRUE + THEN 1 + ELSE 0 + END AS is_nimbus_disabled, + normalized_channel, + normalized_country_code, + sample_id, + ANY_VALUE(ping_info.experiments) AS experiments + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + WHERE + DATE(submission_timestamp) = @submission_date + GROUP BY + client_id, + submission_date, + is_opt_in, + is_opt_out, + normalized_channel, + normalized_country_code, + sample_id, + is_onboarded, + is_nimbus_disabled +), +active AS ( + SELECT + client_info.client_id, + DATE(submission_timestamp) AS submission_date, + COALESCE(SUM(metrics.timespan.glean_baseline_duration.value), 0) / 3600.0 AS active_hours + FROM + `moz-fx-data-shared-prod.firefox_ios.baseline` + WHERE + DATE(submission_timestamp)= @submission_date + GROUP BY + client_info.client_id, + submission_date +), +search AS ( + SELECT + submission_date, + client_id, + search_count AS sap, + ad_click + FROM + `moz-fx-data-shared-prod.search.mobile_search_clients_engines_sources_daily` + WHERE + os = "iOS" + AND normalized_app_name = "Fennec" + AND DATE(submission_date) = @submission_date +), +joined_data AS ( + SELECT + sm.submission_date, + normalized_channel, + normalized_country_code, + sm.client_id, + sm.sample_id, + sm.shopping_product_page_visits, + sm.experiments, + sm.is_opt_in, + sm.is_opt_out, + s.sap, + s.ad_click, + active.active_hours AS active_hours_sum + FROM + shopping_metrics sm + LEFT JOIN + active + ON + active.client_id = sm.client_id + AND active.submission_date = sm.submission_date + LEFT JOIN + search AS s + ON + s.client_id = sm.client_id + AND s.submission_date = sm.submission_date +) +SELECT + * +FROM + joined_data diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/schema.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/schema.yaml new file mode 100644 index 00000000000..6a3eecc656a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_clients_v1/schema.yaml @@ -0,0 +1,58 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: normalized_country_code + type: STRING + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: shopping_product_page_visits + type: INTEGER + mode: NULLABLE +- name: experiments + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: RECORD + mode: NULLABLE + fields: + - name: branch + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: NULLABLE + fields: + - name: enrollment_id + type: STRING + mode: NULLABLE + - name: type + type: STRING + mode: NULLABLE +- name: is_opt_in + type: INTEGER + mode: NULLABLE +- name: is_opt_out + type: INTEGER + mode: NULLABLE +- name: sap + type: INTEGER + mode: NULLABLE +- name: ad_click + type: INTEGER + mode: NULLABLE +- name: active_hours_sum + type: FLOAT + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/metadata.yaml new file mode 100644 index 00000000000..19c1d2bc19a --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/metadata.yaml @@ -0,0 +1,22 @@ +friendly_name: Firefox iOS review checker events data +description: |- + The events table includes telemetry related to the review checker sidebar including user + interaction with the sidebar, open and closed states, and the status of reviews (stale, unavailable, etc.) +owners: +- akommasani@mozilla.com +- betling@mozilla.com +labels: + incremental: true + schedule: daily +scheduling: + dag_name: bqetl_review_checker + task_name: org_mozilla_ios_firefox_review_checker_events__v1 +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + clustering: + fields: + - normalized_channel + - sample_id diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/query.sql b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/query.sql new file mode 100644 index 00000000000..f2476a56983 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/query.sql @@ -0,0 +1,62 @@ +SELECT + DATE(submission_timestamp) AS submission_date, + client_info.client_id AS client_id, + client_info.os AS os, + client_info.os_version AS os_version, + MAX(CASE WHEN name = 'address_bar_feature_callout_displayed' THEN 1 ELSE 0 END) is_address_bar_feature_callout_displayed, + MAX(CASE WHEN name = 'address_bar_icon_clicked' THEN 1 ELSE 0 END) is_address_bar_icon_clicked, + MAX(CASE WHEN name = 'address_bar_icon_displayed' THEN 1 ELSE 0 END) is_address_bar_icon_displayed, + MAX(CASE WHEN name = 'surface_analyze_reviews_none_available_clicked' THEN 1 ELSE 0 END) is_surface_analyze_reviews_none_available_clicked, + MAX(CASE WHEN name = 'surface_closed' THEN 1 ELSE 0 END) is_surface_closed, + MAX(CASE WHEN name = 'surface_displayed' THEN 1 ELSE 0 END) is_surface_displayed, + MAX(CASE WHEN name = 'surface_settings_expand_clicked' THEN 1 ELSE 0 END) is_surface_settings_expand_clicked, + MAX(CASE WHEN name = 'surface_learn_more_clicked' THEN 1 ELSE 0 END) is_surface_learn_more_clicked, + MAX(CASE WHEN name = 'surface_no_review_reliability_available' THEN 1 ELSE 0 END) is_surface_no_review_reliability_available, + MAX(CASE WHEN name = 'surface_onboarding_displayed' THEN 1 ELSE 0 END) is_surface_onboarding_displayed, + MAX(CASE WHEN name = 'surface_opt_in_accepted' THEN 1 ELSE 0 END) is_surface_opt_in_accepted, + MAX(CASE WHEN name = 'surface_reactivated_button_clicked' THEN 1 ELSE 0 END) is_surface_reactivated_button_clicked, + MAX(CASE WHEN name = 'surface_reanalyze_clicked' THEN 1 ELSE 0 END) is_surface_reanalyze_clicked, + MAX(CASE WHEN name = 'surface_powered_by_fakespot_link_clicked' THEN 1 ELSE 0 END) is_surface_powered_by_fakespot_link_clicked, + MAX(CASE WHEN name = 'surface_show_more_recent_reviews_clicked' THEN 1 ELSE 0 END) is_surface_show_more_reviews_button_clicked, + MAX(CASE WHEN name = 'surface_show_privacy_policy_clicked' THEN 1 ELSE 0 END) is_surface_show_privacy_policy_clicked, + MAX(CASE WHEN name = 'surface_show_quality_explainer_clicked' THEN 1 ELSE 0 END) is_surface_show_quality_explainer_url_clicked, + MAX(CASE WHEN name = 'surface_show_terms_clicked' THEN 1 ELSE 0 END) is_surface_show_terms_clicked, + MAX(CASE WHEN name = 'surface_stale_analysis_shown' THEN 1 ELSE 0 END) is_surface_stale_analysis_shown, + MAX( + CASE + WHEN name IN ( + 'surface_analyze_reviews_none_available_clicked', + 'surface_learn_more_clicked', + 'surface_no_review_reliability_available', + 'surface_reactivated_button_clicked', + 'surface_reanalyze_clicked', + 'surface_settings_expand_clicked', + 'surface_show_more_recent_reviews_clicked', + 'surface_show_privacy_policy_clicked', + 'surface_show_quality_explainer_clicked', + 'surface_show_terms_clicked' + ) + THEN 1 + ELSE 0 + END + ) is_engaged_with_sidebar, + ANY_VALUE(ping_info.experiments) AS experiments, + normalized_channel, + normalized_country_code, + sample_id, + mozfun.norm.truncate_version(client_info.app_display_version, "major") AS app_version, +FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` AS e, + UNNEST(events) +WHERE + DATE(submission_timestamp) = @submission_date + AND category = 'shopping' +GROUP BY + DATE(submission_timestamp), + client_info.client_id, + normalized_channel, + normalized_country_code, + sample_id, + os, + os_version, + app_version diff --git a/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/schema.yaml b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/schema.yaml new file mode 100644 index 00000000000..55c3afd21c8 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/review_checker_events_v1/schema.yaml @@ -0,0 +1,109 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: client_id + type: STRING + mode: NULLABLE +- name: os + type: STRING + mode: NULLABLE +- name: os_version + type: STRING + mode: NULLABLE +- name: is_address_bar_feature_callout_displayed + type: INTEGER + mode: NULLABLE +- name: is_address_bar_icon_clicked + type: INTEGER + mode: NULLABLE +- name: is_address_bar_icon_displayed + type: INTEGER + mode: NULLABLE +- name: is_surface_analyze_reviews_none_available_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_closed + type: INTEGER + mode: NULLABLE +- name: is_surface_displayed + type: INTEGER + mode: NULLABLE +- name: is_surface_settings_expand_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_learn_more_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_no_review_reliability_available + type: INTEGER + mode: NULLABLE +- name: is_surface_onboarding_displayed + type: INTEGER + mode: NULLABLE +- name: is_surface_opt_in_accepted + type: INTEGER + mode: NULLABLE +- name: is_surface_reactivated_button_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_reanalyze_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_powered_by_fakespot_link_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_more_reviews_button_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_privacy_policy_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_quality_explainer_url_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_show_terms_clicked + type: INTEGER + mode: NULLABLE +- name: is_surface_stale_analysis_shown + type: INTEGER + mode: NULLABLE +- name: is_engaged_with_sidebar + type: INTEGER + mode: NULLABLE +- name: experiments + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: RECORD + mode: NULLABLE + fields: + - name: branch + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: NULLABLE + fields: + - name: enrollment_id + type: STRING + mode: NULLABLE + - name: type + type: STRING + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: normalized_country_code + type: STRING + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE +- name: app_version + type: NUMERIC + mode: NULLABLE diff --git a/sql_generators/review_checker/__init__.py b/sql_generators/review_checker/__init__.py deleted file mode 100644 index 92b81101abb..00000000000 --- a/sql_generators/review_checker/__init__.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Generate review checker events/clients/microsruvey data per app.""" -import os -from enum import Enum -from pathlib import Path - -import click -import yaml -from jinja2 import Environment, FileSystemLoader - -from bigquery_etl.cli.utils import use_cloud_function_option -from bigquery_etl.format_sql.formatter import reformat -from bigquery_etl.util.common import render, write_sql - -THIS_PATH = Path(os.path.dirname(__file__)) - - -class Browsers(Enum): - """Enumeration with browser names and equivalent dataset names.""" - - firefox_desktop = "Firefox Desktop" - - -@click.command() -@click.option( - "--output-dir", - "--output_dir", - help="Output directory generated SQL is written to", - type=click.Path(file_okay=False), - default="sql", -) -@click.option( - "--target-project", - "--target_project", - help="Google Cloud project ID", - default="moz-fx-data-shared-prod", -) -@use_cloud_function_option -def generate(target_project, output_dir, use_cloud_function): - """Generate per-app queries, views and metadata for review checker aggregates. - - The parent folders will be created if not existing and existing files will be overwritten. - """ - with open(THIS_PATH / "templates/templating.yaml", "r") as f: - template_config = yaml.safe_load(f) - - output_dir = Path(output_dir) / target_project - for query, args in template_config["queries"].items(): - template_query_dir = THIS_PATH / "templates" / query - env = Environment(loader=FileSystemLoader(str(THIS_PATH / "templates" / query))) - query_template = env.get_template("query.sql") - metadata_template = "metadata.yaml" - view_template = env.get_template("view.sql") - for browser in Browsers: - print(browser) - query_sql = reformat( - query_template.render( - project_id=target_project, - app_value=browser.value, - app_name=browser.name, - ) - ) - - write_sql( - output_dir=output_dir, - full_table_id=f"{target_project}.{browser.name}_derived.{query}_v1", - basename="query.sql", - sql=query_sql, - skip_existing=False, - ) - - write_sql( - output_dir=output_dir, - full_table_id=f"{target_project}.{browser.name}_derived.{query}_v1", - basename="metadata.yaml", - sql=render( - metadata_template, - template_folder=THIS_PATH / "templates" / query, - app_value=browser.value, - app_name=browser.name, - format=False, - ), - skip_existing=False, - ) - - write_sql( - output_dir=output_dir, - full_table_id=f"{target_project}.{browser.name}.{query}", - basename="view.sql", - sql=reformat( - view_template.render( - project_id=target_project, - app_name=browser.name, - ) - ), - skip_existing=False, - ) diff --git a/sql_generators/review_checker/templates/review_checker_clients/view.sql b/sql_generators/review_checker/templates/review_checker_clients/view.sql deleted file mode 100644 index 82a4c6115d8..00000000000 --- a/sql_generators/review_checker/templates/review_checker_clients/view.sql +++ /dev/null @@ -1,8 +0,0 @@ ---- User-facing view. Generated via sql_generators.active_users. -CREATE OR REPLACE VIEW - `{{ project_id }}.{{ app_name }}.review_checker_clients` -AS -SELECT - * -FROM - `{{ project_id }}.{{ app_name }}_derived.review_checker_clients_v1` diff --git a/sql_generators/review_checker/templates/review_checker_events/view.sql b/sql_generators/review_checker/templates/review_checker_events/view.sql deleted file mode 100644 index c03dbf1c05d..00000000000 --- a/sql_generators/review_checker/templates/review_checker_events/view.sql +++ /dev/null @@ -1,8 +0,0 @@ ---- User-facing view. Generated via sql_generators.active_users. -CREATE OR REPLACE VIEW - `{{ project_id }}.{{ app_name }}.review_checker_events` -AS -SELECT - * -FROM - `{{ project_id }}.{{ app_name }}_derived.review_checker_events_v1` diff --git a/sql_generators/review_checker/templates/review_checker_microsurvey/view.sql b/sql_generators/review_checker/templates/review_checker_microsurvey/view.sql deleted file mode 100644 index 071dcd72c7a..00000000000 --- a/sql_generators/review_checker/templates/review_checker_microsurvey/view.sql +++ /dev/null @@ -1,8 +0,0 @@ ---- User-facing view. Generated via sql_generators.active_users. -CREATE OR REPLACE VIEW - `{{ project_id }}.{{ app_name }}.review_checker_microsurvey` -AS -SELECT - * -FROM - `{{ project_id }}.{{ app_name }}_derived.review_checker_microsurvey_v1` diff --git a/sql_generators/review_checker/templates/templating.yaml b/sql_generators/review_checker/templates/templating.yaml deleted file mode 100644 index 76d10061970..00000000000 --- a/sql_generators/review_checker/templates/templating.yaml +++ /dev/null @@ -1,9 +0,0 @@ -queries: - review_checker_clients: - per_app: true - review_checker_events: - per_app: true - review_checker_microsurvey: - per_app: true -applications: - - firefox_desktop \ No newline at end of file