Urlbar events: nested (long) instead of wide (#4373)

* feat: urlbar events final release * feat: new result types * feat: add interaction and group * fix: date * fix: use BQ builtin for UUIDs * Add the view_v2' * Add new table to the DAG * fix CI error fix ci error * remove teon brooks * Incorporate feedback by Curtis Incorporate feedback from Curtis --------- Co-authored-by: Alekhya Kommasani <[email protected]> Co-authored-by: Alekhya <[email protected]>
mozilla · Dec 11, 2023 · 7b051a7 · 7b051a7
1 parent c5f91ff
commit 7b051a7
Show file tree

Hide file tree

Showing 6 changed files with 163 additions and 5 deletions.
diff --git a/dags.yaml b/dags.yaml
@@ -711,7 +711,6 @@ bqetl_urlbar:
       - "[email protected]"
       - "[email protected]"
       - "[email protected]"
-      - "[email protected]"
     retries: 2
     retry_delay: 30m
   tags:

diff --git a/dags/bqetl_urlbar.py b/dags/bqetl_urlbar.py
@@ -31,7 +31,6 @@
         "[email protected]",
         "[email protected]",
         "[email protected]",
-        "[email protected]",
     ],
     "depends_on_past": False,
     "retry_delay": datetime.timedelta(seconds=1800),
@@ -61,7 +60,24 @@
             "[email protected]",
             "[email protected]",
             "[email protected]",
-            "[email protected]",
+            "[email protected]",
+        ],
+        date_partition_parameter="submission_date",
+        depends_on_past=False,
+    )
+
+    firefox_desktop_urlbar_events__v2 = bigquery_etl_query(
+        task_id="firefox_desktop_urlbar_events__v2",
+        destination_table="urlbar_events_v2",
+        dataset_id="firefox_desktop_derived",
+        project_id="moz-fx-data-shared-prod",
+        owner="[email protected]",
+        email=[
+            "[email protected]",
+            "[email protected]",
+            "[email protected]",
+            "[email protected]",
+            "[email protected]",
             "[email protected]",
         ],
         date_partition_parameter="submission_date",
@@ -98,6 +114,8 @@
 
     firefox_desktop_urlbar_events__v1.set_upstream(wait_for_copy_deduplicate_all)
 
+    firefox_desktop_urlbar_events__v2.set_upstream(wait_for_copy_deduplicate_all)
+
     wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskSensor(
         task_id="wait_for_telemetry_derived__clients_daily_joined__v1",
         external_dag_id="bqetl_main_summary",

diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop/urlbar_events_temp_v2/view.sql b/sql/moz-fx-data-shared-prod/firefox_desktop/urlbar_events_temp_v2/view.sql
@@ -0,0 +1,8 @@
+--- User-facing view. Generated via sql_generators.active_users.
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.firefox_desktop.urlbar_events_temp_v2`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.firefox_desktop_derived.urlbar_events_v2`
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/metadata.yaml
@@ -0,0 +1,21 @@
+friendly_name: Urlbar Events
+description: |-
+  Each row in this table represents one engagement (https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/metrics/urlbar_engagement)
+  or abandonment(https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/metrics/urlbar_abandonment) event. A urlbar session ends
+  when the urlbar dropdown menu closes, however, not all events mark the end of a urlbar session. To identify the events marking the end of a
+  urlbar session, filter on the field is_terminal = true.
+owners:
+- [email protected]
+- [email protected]
+- [email protected]
+labels:
+  incremental: true
+  schedule: daily
+scheduling:
+  dag_name: bqetl_urlbar
+  task_name: firefox_desktop_urlbar_events__v2
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql
@@ -0,0 +1,110 @@
+CREATE TEMP FUNCTION enumerated_array(results ARRAY<STRING>, _groups ARRAY<STRING>) AS (
+  ARRAY(
+    SELECT
+      STRUCT(
+        off + 1 AS position,
+        r AS result_type,
+        `mozfun.norm.result_type_to_product_name`(r) AS product_result_type,
+        _groups[SAFE_OFFSET(off)] AS result_group
+      )
+    FROM
+      UNNEST(results) AS r
+      WITH OFFSET off
+  )
+);
+
+CREATE TEMP FUNCTION get_event_action(event_name STRING, engagement_type STRING) AS (
+  CASE
+    WHEN event_name = 'engagement'
+      AND (engagement_type IN ("click", "drop_go", "enter", "go_button", "paste_go"))
+      THEN 'engaged'
+    WHEN event_name = 'abandonment'
+      THEN 'abandoned'
+    WHEN event_name = 'engagement'
+      AND (engagement_type NOT IN ("click", "drop_go", "enter", "go_button", "paste_go"))
+      THEN "annoyance"
+    ELSE NULL
+  END
+);
+
+CREATE TEMP FUNCTION get_is_terminal(selected_result STRING, engagement_type STRING) AS (
+  --events where the urlbar dropdown menu remains open (i.e., the urlbar session did not end)
+  COALESCE(
+    NOT (selected_result = 'tab_to_search' AND engagement_type IN ('click', 'enter', 'go_button'))
+    AND NOT (
+      selected_result = 'tip_dismissal_acknowledgement'
+      AND engagement_type IN ('click', 'enter')
+    )
+    AND NOT (
+      engagement_type IN (
+        'dismiss',
+        'inaccurate_location',
+        'not_interested',
+        'not_relevant',
+        'show_less_frequently'
+      )
+    ),
+    TRUE
+  )
+);
+
+WITH events_unnested AS (
+  SELECT
+    DATE(submission_timestamp) AS submission_date,
+    client_info.client_id AS glean_client_id,
+    metrics.uuid.legacy_telemetry_client_id AS legacy_telemetry_client_id,
+    sample_id,
+    name AS event_name,
+    timestamp AS event_timestamp,
+    GENERATE_UUID() AS event_id,
+    ping_info.experiments,
+    ping_info.seq,
+    normalized_channel,
+    normalized_country_code,
+    `moz-fx-data-shared-prod`.udf.normalize_search_engine(
+      mozfun.map.get_key(extra, "search_engine_default_id")
+    ) AS normalized_engine,
+    COALESCE(metrics.boolean.urlbar_pref_suggest_data_collection, FALSE) AS pref_data_collection,
+    COALESCE(metrics.boolean.urlbar_pref_suggest_sponsored, FALSE) AS pref_sponsored_suggestions,
+    COALESCE(metrics.boolean.urlbar_pref_suggest_nonsponsored, FALSE) AS pref_fx_suggestions,
+    mozfun.map.get_key(extra, "engagement_type") AS engagement_type,
+    mozfun.map.get_key(extra, "interaction") AS interaction,
+    CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed,
+    CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results,
+  --If 0, then no result was selected.
+    NULLIF(CAST(mozfun.map.get_key(extra, "selected_position") AS int), 0) AS selected_position,
+    mozfun.map.get_key(extra, "selected_result") AS selected_result,
+    enumerated_array(
+      SPLIT(mozfun.map.get_key(extra, "results"), ','),
+      SPLIT(mozfun.map.get_key(extra, "groups"), ',')
+    ) AS results,
+  FROM
+    `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1`,
+    UNNEST(events) AS event
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+    AND event.category = 'urlbar'
+    AND event.name IN ('engagement', 'abandonment')
+)
+SELECT
+  *,
+  `mozfun.norm.result_type_to_product_name`(selected_result) AS product_selected_result,
+  get_event_action(event_name, engagement_type) AS event_action,
+  get_is_terminal(selected_result, engagement_type) AS is_terminal,
+  CASE
+    WHEN get_event_action(event_name, engagement_type) IN ('engaged', 'annoyance')
+      THEN selected_result
+    ELSE NULL
+  END AS engaged_result_type,
+  CASE
+    WHEN get_event_action(event_name, engagement_type) IN ('engaged', 'annoyance')
+      THEN `mozfun.norm.result_type_to_product_name`(selected_result)
+    ELSE NULL
+  END AS product_engaged_result_type,
+  CASE
+    WHEN get_event_action(event_name, engagement_type) = 'annoyance'
+      THEN engagement_type
+    ELSE NULL
+  END AS annoyance_signal_type,
+FROM
+  events_unnested
diff --git a/sql/mozfun/norm/result_type_to_product_name/udf.sql b/sql/mozfun/norm/result_type_to_product_name/udf.sql
@@ -3,6 +3,9 @@ CREATE OR REPLACE FUNCTION norm.result_type_to_product_name(
 )
 RETURNS STRING AS (
     CASE
+    WHEN res IN ('autofill_origin', 'autofill_url') THEN 'autofill'
+    WHEN res IN ('addon') THEN 'xchannels_add_on'
+    WHEN res IN ('rs_amo') THEN 'suggest_add_on'
     WHEN res IN ('search_suggest', 'search_history', 'search_suggest_rich') THEN 'default_partner_search_suggestion'
     WHEN res IN ('search_engine') THEN 'search_engine'
     WHEN res IN ('trending_search', 'trending_search_rich') THEN 'trending_suggestion'
@@ -11,15 +14,14 @@ RETURNS STRING AS (
     WHEN res IN ('tab') THEN 'open_tab'
     WHEN res IN ('merino_adm_sponsored', 'rs_adm_sponsored', 'suggest_sponsor') THEN 'admarketplace_sponsored'
     WHEN res IN ('merino_top_picks') THEN 'navigational'
-    WHEN res IN ('addon', 'rs_amo') THEN 'add_on'
     WHEN res IN ('rs_adm_nonsponsored',
     'merino_adm_nonsponsored',
     'suggest_non_sponsor') THEN 'wikipedia_enhanced'
     WHEN res IN ('dynamic_wikipedia', 'merino_wikipedia') THEN 'wikipedia_dynamic'
     WHEN res IN ('weather') THEN 'weather'
     WHEN res IN ( 'action', 'intervention_clear', 'intervention_refresh', 'intervention_unknown', 'intervention_update' ) THEN 'quick_action'
     WHEN res IN ('rs_pocket') THEN 'pocket_collection'
-    ELSE NULL
+    ELSE 'other'
     END
 );