MPP-3352: Add first Glean metrics to measure email mask usage

.circleci/config.yml

@@ -378,7 +378,7 @@ jobs:
         description: "What command should the job run?"
         default: "pytest"
         type: enum
-        enum: ["pytest", "black", "mypy", "build_email_tracker_lists"]
+        enum: ["pytest", "black", "mypy", "build_email_tracker_lists", "build_glean", "check_glean"]
       test_results_filename:
         description: "What is the name of the jUnit XML test output? (Optional)"
         default: ""
@@ -697,6 +697,11 @@ workflows:
           test_results_filename: pytest-iq-enabled.xml
           filters: *default_filters
 
+      - python_job:
+          name: check generated glean_parser files
+          command: check_glean
+          filters: *default_filters
+
       - test_frontend:
           requires:
             - build_frontend

.circleci/python_job.bash

@@ -13,6 +13,8 @@
 : ${TEST_DB_NAME:=test.sqlite3}
 : ${TEST_DB_URL:=sqlite:///test.sqlite3}
 : ${DATABASE_ENGINE:=sqlite}
+: ${GLEAN_OUTPUT_FOLDER:=privaterelay/glean}
+: ${GLEAN_INPUT_YAML:=telemetry/glean/relay-server-metrics.yaml}
 
 # Run black to check Python format
 function run_black {
@@ -73,13 +75,85 @@ function run_build_email_tracker_lists {
     cp /home/circleci/project/emails/tracker_lists/level-two-trackers.json /tmp/workspace/email-trackers/
 }
 
+
+
+# Run commands to build the backend Glean code
+function run_build_glean {
+    local OUTPUT_FOLDER=${1:-$GLEAN_OUTPUT_FOLDER}
+    local INPUT_YAML=${2:-$GLEAN_INPUT_YAML}
+    set -x
+    glean_parser translate --format python_server --output ${OUTPUT_FOLDER} ${INPUT_YAML}
+    black ${OUTPUT_FOLDER}
+    case "$OSTYPE" in
+        darwin* | bsd*)
+            echo "Using BSD sed";
+            sed -i '' \
+                -e 's/^AUTOGENERATED BY glean_parser \(.* DO NOT EDIT.\) DO NOT COMMIT.$/AUTOGENERATED BY glean_parser \1 To recreate, run:\n\nbash .circleci\/python_job.bash run build_glean/' \
+                ${OUTPUT_FOLDER}/server_events.py;;
+        linux*)
+            echo "Using GNU sed";
+            sed -i \
+                -e 's/^AUTOGENERATED BY glean_parser \(.* DO NOT EDIT.\) DO NOT COMMIT.$/AUTOGENERATED BY glean_parser \1 To recreate, run:\n\nbash .circleci\/python_job.bash run build_glean/' \
+                ${OUTPUT_FOLDER}/server_events.py;;
+        *)
+            echo "Unknown OSTYPE '${OSTYPE}'";;
+    esac
+}
+
+# Run commands to check if the backend Glean code has changed
+function run_check_glean {
+    GLEAN_TEST_FOLDER=`mktemp -d`
+    echo "Created temporary folder $GLEAN_TEST_FOLDER"
+    echo "Re-running glean_parser..."
+    echo
+    run_build_glean $GLEAN_TEST_FOLDER
+    set +x
+
+    echo
+    echo "Checking new files against checked-in files..."
+    local HAS_DIFFS=0
+    local HAS_FILES=0
+    local EXIT_CODE=0
+    for FILENAME in `find ${GLEAN_TEST_FOLDER} -type f -exec basename \{\} \;`
+    do
+        HAS_FILES=1
+        if [[ 0 -ne `cmp --silent "${GLEAN_TEST_FOLDER}/${FILENAME}" "${GLEAN_OUTPUT_FOLDER}/${FILENAME}"` ]]
+        then
+            HAS_DIFFS=1
+        fi
+    done
+    if [ $HAS_FILES -eq 0 ]
+    then
+        echo "glean_parser did not generate any files!"
+        EXIT_CODE=1
+    fi
+    if [ $HAS_DIFFS -eq 1 ]
+    then
+        echo "*** Differences detected - need to re-run glean_parser ***"
+        diff $GLEAN_OUTPUT_FOLDER $GLEAN_TEST_FOLDER
+        EXIT_CODE=1
+    fi
+    if [ "$GLEAN_TEST_FOLDER" != "" ]
+    then
+        echo "Removing temporary folder $GLEAN_TEST_FOLDER"
+        rm -rf $GLEAN_TEST_FOLDER
+    fi
+    if [ $EXIT_CODE -eq 0 ]
+    then
+        echo "✓ Files are identical"
+    else
+        exit 1
+    fi
+}
+
+
 # Run a command by name
 # $1 - The command to run - black, mypy, pytest, or build_email_tracker_lists
 # Remaining arguments are passed to the run_COMMAND function
 function run_command {
     local COMMAND=${1:-}
     case $COMMAND in
-        black | mypy | pytest | build_email_tracker_lists)
+        black | mypy | pytest | build_email_tracker_lists | build_glean | check_glean)
             :;;
         "")
             echo "No command passed - '$COMMAND'"
@@ -142,3 +216,8 @@ function switch_to_production {
     git status
     pip install -r requirements.txt
 }
+
+if [ "$1" == "run" ]
+then
+    run_command "$2"
+fi