Coverage for rust agent as github action.

Implement github action which gather code coverage of PR against master branch. And after merge it gather code coverage again. Add helper script which gather code coverage from testing farm. And update e2e plan where run tasks for measuring code coverage. Signed-off-by: Patrik Koncity <[email protected]>
keylime · Feb 27, 2023 · 5051383 · 5051383
1 parent 1b8cd8b
commit 5051383
Show file tree

Hide file tree

Showing 5 changed files with 267 additions and 0 deletions.
diff --git a/.github/workflows/submit-HEAD-coverage.yaml b/.github/workflows/submit-HEAD-coverage.yaml
@@ -0,0 +1,36 @@
+name: Upload code coverage for a merged PR to codecov.io
+
+on:
+  push:
+    branches:
+    - master
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    name: Submit code coverage from merged PR
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install testing-farm script
+      run: pip3 -v install tft-cli
+    - name: Run tests on Testing Farm
+      run: testing-farm request --context distro=fedora-37 --arch x86_64 --compose Fedora-37 --plan '/e2e' -e UPLOAD_COVERAGE=1 2>&1 | tee tt_output
+      env:
+        TESTING_FARM_API_TOKEN: ${{ secrets.TESTING_FARM_API_TOKEN }}
+    - name: Find PR Packit tests to finish and download e2e_coverage.txt and upstream_coverage.xml coverage files.
+      run: grep -q 'tests passed' tt_output && sleep 20 && scripts/download_packit_coverage.sh --testing-farm-log tt_output
+      env:
+        MAX_DURATION: 120
+        SLEEP_DELAY: 20
+    - name: List downloaded files.
+      run: ls 
+    - name: Upload e2e_coverage report to Codecov with GitHub Action.
+      uses: codecov/codecov-action@v2
+      with:
+        files: e2e_coverage.txt
+        flags: e2e_coverage
+    - name: Upload upstream_coverage report to Codecov with GitHub Action.
+      uses: codecov/codecov-action@v2
+      with:
+        files: upstream_coverage.xml
+        flags: upstream-unit-tests
diff --git a/.github/workflows/submit-PR-coverage.yaml b/.github/workflows/submit-PR-coverage.yaml
@@ -0,0 +1,31 @@
+name: PR code coverage measurement on codecov.io
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    name: Submit code coverage from Packit tests
+    defaults:
+      run:
+        working-directory: scripts
+    steps:
+    - uses: actions/checkout@v3
+    - name: Wait for Packit tests to finish and download e2e_coverage.txt and upstream_coverage.xml files.
+      run: ./download_packit_coverage.sh
+      env:
+        MAX_DURATION: 5400
+        SLEEP_DELAY: 120
+    - name: List downloaded files.
+      run: ls
+    - name: Upload e2e_coverage.txt report to Codecov with GitHub Action.
+      uses: codecov/codecov-action@v2
+      with:
+        files: e2e_coverage.txt
+        flags: e2e-testsuite
+    - name: Upload upstream_coverage.xml report to Codecov with GitHub Action.
+      uses: codecov/codecov-action@v2
+      with:
+        files: upstream_coverage.xml
+        flags: upstream-unit-tests
diff --git a/codecov.yml b/codecov.yml
@@ -0,0 +1,25 @@
+codecov:
+  notify:
+    after_n_builds: 2
+
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+
+comment:
+  layout: "flags,files"
+  behavior: default
+  require_changes: false
+  require_base: no
+  after_n_builds: 1
+
+ignore:
+    - "**/lib.rs"
+
+flags:
+  e2e-testsuite:
+    carryforward: false
+  upstream-unit-tests:
+    carryforward: false
diff --git a/packit-ci.fmf b/packit-ci.fmf
@@ -5,6 +5,7 @@
   environment:
     TPM_BINARY_MEASUREMENTS: /var/tmp/binary_bios_measurements
     RUST_IMA_EMULATOR: 1
+    KEYLIME_RUST_CODE_COVERAGE: 1
 
   context:
     swtpm: yes
@@ -53,6 +54,8 @@
        - /functional/tpm-issuer-cert-using-ecc
        - /functional/tpm_policy-sanity-on-localhost
        - /functional/use-multiple-ima-sign-verification-keys
+       - /upstream/run_rust_keylime_tests
+       - /setup/generate_usptream_rust_keylime_code_coverage
 
   adjust:
    # prepare step adjustments
@@ -61,6 +64,13 @@
       - how: shell
         script:
          - yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+     # disable code coverage measurement everywhere except F37 and CS9
+   - when: distro != fedora-37
+     environment+:
+        KEYLIME_RUST_CODE_COVERAGE: 0
+     discover+:
+        test-:
+         - /setup/generate_usptream_rust_keylime_code_coverage
 
   execute:
     how: tmt
diff --git a/scripts/download_packit_coverage.sh b/scripts/download_packit_coverage.sh
@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# There are 3 options how to tell this script where to start
+# --artifacts-url - Testing Farm artifacts URL, provided by testing-farm script
+# --testing-farm-log - Log of 'testing-farm request' command from where artifacts URL will be parsed
+# --github-sha - PR merge commit provided by GitHub, here we will try to get artifacts URL using GitHub API
+
+if [ "$1" == "--artifacts-url" -a -n "$2" ]; then
+    TF_ARTIFACTS_URL="$2"
+elif [ "$1" == "--testing-farm-log" -a -n "$2" ]; then
+    TT_LOG="$2"
+elif [ "$1" == "--github-sha" -a -n "$2" ]; then
+    GITHUB_SHA="$2"
+elif [ -n "$GITHUB_SHA" ]; then
+    :
+else
+    echo "Neither --github-sha nor --artifacts-url nor --testing-farm-log arguments were provided"
+    exit 1
+fi
+
+##############################################
+# initial configuration, adjust when necessary
+##############################################
+
+# maximum duration of the task in seconds
+MAX_DURATION="${MAX_DURATION:-5400}"  # 90 minutes
+
+# delay in seconds before doing another URL read
+# should not be too short not to exceed GitHub API quota
+SLEEP_DELAY="${SLEEP_DELAY:-120}"
+
+# github user/project we are going to work with
+PROJECT="keylime/rust-keylime"
+
+# TF_JOB_DESC points to a Testing farm job that does code coverage measurement and 
+# uploads coverage XML files to a web drive
+# currently we are doing that in a job running tests on Fedora-37
+TF_JOB_DESC="testing-farm:fedora-37-x86_64"
+TF_TEST_OUTPUT="/setup/generate_usptream_rust_keylime_code_coverage/output.txt"
+TF_ARTIFACTS_URL_PREFIX="https://artifacts.dev.testing-farm.io"
+
+GITHUB_API_PREFIX_URL="https://api.github.com/repos/${PROJECT}"
+
+WEBDRIVE_URL="https://(transfer.sh|free.keep.sh)"
+
+##################################
+# no need to change anything below
+##################################
+
+# build GITHUB_API_URLs
+GITHUB_API_COMMIT_URL="${GITHUB_API_PREFIX_URL}/commits"
+DURATION=0
+
+TMPFILE=$( mktemp )
+
+# run API call and parse the required value
+# repeat until we get the value or exceed job duration
+# URL - API URL
+# JQ_REF - code for jq that will be used for JSON parsing
+# ERROR_MSG - error message to print in case we fail to parse the value
+# EXP_VALUE - expected value (used e.g. when waiting for job completion)
+function do_GitHub_API_call() {
+    local URL="$1"
+    local JQ_REF="$2"
+    local ERROR_MSG="$3"
+    local EXP_VALUE="$4"
+    local VALUE=''
+
+    while [ -z "${VALUE}" -o \( -n "${EXP_VALUE}" -a "${VALUE}" != "${EXP_VALUE}" \) ] && [ ${DURATION} -lt ${MAX_DURATION} ]; do
+        if [ "$URL" != "-" ]; then  # when URL='-', we reuse data downloaded previously
+            curl --retry 5 -s -H "Accept: application/vnd.github.v3+json" "$URL" &> ${TMPFILE}
+        fi
+        VALUE=$( cat ${TMPFILE} | jq "${JQ_REF}" | sed 's/"//g' )
+        if [ -z "${VALUE}" ] || [ -n "${EXP_VALUE}" -a "${VALUE}" != "${EXP_VALUE}" ]; then
+            if [ -z "${ERROR_MSG}" ]; then
+                echo "Warning: Failed to read data using GitHub API, trying again after ${SLEEP_DELAY} seconds" 1>&2
+            else
+                echo "$ERROR_MSG" 1>&2
+            fi
+            sleep ${SLEEP_DELAY}
+            DURATION=$(( ${DURATION}+${SLEEP_DELAY} ))
+        fi
+    done
+
+    if [ ${DURATION} -ge ${MAX_DURATION} ]; then
+         echo "Error: Maximum job duration exceeded. Terminating" 1>&2
+         exit 9
+    fi
+
+    echo $VALUE
+}
+
+
+# if the GitHub Action has been triggered by a PR, 
+# we need to find Testing farm test results through GitHub API
+if [ -n "${GITHUB_SHA}" -a -z "${TF_ARTIFACTS_URL}" -a -z "${TT_LOG}" ]; then
+
+    echo "Trying to find Testing Farm / Packig CI test results using GitHub API"
+
+    echo "Fist I need to find the respective PR commit"
+    GITHUB_API_SHA_URL="${GITHUB_API_COMMIT_URL}/${GITHUB_SHA}"
+
+    # Now we try to parse URL of Testing farm job from GITHUB_API_RUNS_URL page
+    GITHUB_PR_SHA=$( do_GitHub_API_call "${GITHUB_API_SHA_URL}" \
+                                     ".parents[1].sha" \
+                                     "Failed to parse PR commit from ${GITHUB_API_RUNS_URL}, trying again after ${SLEEP_DELAY} seconds..." )
+    echo "GITHUB_PR_SHA=${GITHUB_PR_SHA}"
+
+    echo "Now we read check-runs details"
+    # build GITHUB_API_RUNS_URL using the COMMIT
+    GITHUB_API_RUNS_URL="${GITHUB_API_COMMIT_URL}/${GITHUB_PR_SHA}/check-runs?check_name=${TF_JOB_DESC}"
+    echo "GITHUB_API_RUNS_URL=${GITHUB_API_RUNS_URL}"
+
+    # Now we try to parse URL of Testing farm job from GITHUB_API_RUNS_URL page
+    TF_ARTIFACTS_URL=$( do_GitHub_API_call "${GITHUB_API_RUNS_URL}" \
+                                     ".check_runs[0] | .output.summary | match(\"${TF_ARTIFACTS_URL_PREFIX}[^ ]*\") | .string" \
+                                     "Failed to parse Testing Farm job ${TF_JOB_DESC} URL from ${GITHUB_API_RUNS_URL}, trying again after ${SLEEP_DELAY} seconds..." )
+    echo "TF_ARTIFACTS_URL=${TF_ARTIFACTS_URL}"
+
+    # now we wait for the Testing farm job to finish
+    TF_STATUS=$( do_GitHub_API_call "${GITHUB_API_RUNS_URL}" \
+                                    '.check_runs[0] | .status' \
+                                    "Testing Farm job ${TF_JOB_DESC} hasn't completed yet, trying again after ${SLEEP_DELAY} seconds..." \
+                                    "completed" )
+    echo "TF_STATUS=${TF_STATUS}"
+
+fi
+
+# if we were provided with testing-farm command log
+# we will parse artifacts from the log
+if [ -n "${TT_LOG}" ]; then
+    cat ${TT_LOG}
+    TF_ARTIFACTS_URL=$( egrep -o "${TF_ARTIFACTS_URL_PREFIX}[^ ]*" ${TT_LOG} )
+fi
+
+# now we have TF_ARTIFACTS_URL so we can proceed with the download
+echo "TF_ARTIFACTS_URL=${TF_ARTIFACTS_URL}"
+
+TF_TESTLOG=$( curl --retry 5 ${TF_ARTIFACTS_URL}/results.xml | egrep -o "${TF_ARTIFACTS_URL}.*${TF_TEST_OUTPUT}" )
+echo "TF_TESTLOG=${TF_TESTLOG}"
+
+# parse the URL of coverage txt file on WEBDRIVE_URL and download it
+curl --retry 5 -s "${TF_TESTLOG}" &> ${TMPFILE}
+echo "TMPFILE=${TMPFILE}"
+# probabbly rewrite, different hardcoded files, need to figureout how to export
+
+#download test coverage 
+COVERAGE_URL=$( grep "e2e_coverage.txt report is available at" ${TMPFILE} | egrep -o "${WEBDRIVE_URL}.*\.txt" )
+echo "COVERAGE_URL=${COVERAGE_URL}"
+if [ -z "${COVERAGE_URL}" ]; then
+    echo "Could not parse e2e_coverage.txt URL at ${WEBDRIVE_URL} from test log ${TF_TESTLOG}"
+    exit 5
+fi
+# download the file
+curl --retry 5 -L -O ${COVERAGE_URL}
+#download upstream test coverage 
+COVERAGE_URL=$( grep "upstream_coverage.xml report is available at" ${TMPFILE} | egrep -o "${WEBDRIVE_URL}.*\.xml" )
+echo "COVERAGE_URL=${COVERAGE_URL}"
+if [ -z "${COVERAGE_URL}" ]; then
+    echo "Could not parse upstream_coverage.xml at ${WEBDRIVE_URL} from test log ${TF_TESTLOG}"
+    exit 5
+fi
+# download the file
+curl --retry 5 -L -O ${COVERAGE_URL}
+rm ${TMPFILE}