spread-tests.yaml

on:
  workflow_call:
    inputs:
      runs-on:
        description: 'A json list of tags to indicate which runner to use'
        required: true
        type: string
      group:
        description: 'The name of the group of backends, systems, tests, and rules'
        required: true
        type: string
      backend:
        description: 'The spread backend to use (for possible values, check spread.yaml). This cannot be empty'
        required: true
        type: string
      systems:
        description: 'The spread system(s) to use (for possible values, check spread.yaml). If more than one, separate them with a space. To run all, write ALL. To run none, write NONE'
        required: true
        type: string
      tasks:
        description: 'The spread tasks to run. It may be a space-separated list and may contain directories of many tasks or individual ones'
        required: true
        type: string
      rules:
        description: 'The rule .yaml file to use (found under tests/lib/spread/rules) for test discovery'
        required: true
        type: string
      is-fundamental:
        description: 'If true, then will mark results as from fundamental systems when uploading to Grafana'
        required: false
        type: boolean
        default: false
      use-snapd-snap-from-master:
        description: 'If true, will use the snapd snap built on the master branch'
        required: false
        type: boolean
        default: false
      spread-snapd-deb-from-repo:
        description: 'If true, will use the snapd package from the repository when possible'
        required: false
        type: string
      spread-experimental-features:
        description: 'Comma-separated list of experimental snapd features to enable with: snap set system "experimental.<feature-name>=true"'
        required: false
        type: string


jobs:
  run-spread:
    env:
      SPREAD_EXPERIMENTAL_FEATURES: ${{ inputs.spread-experimental-features }}
      
    runs-on: ${{ fromJSON(inputs.runs-on) }}
    steps:

    - name: Verify inputs
      run: |
          if [ -z "${{ inputs.backend }}" ]; then
            echo "You must specify a backend."
            exit 1
          fi
          if [ -z "${{ inputs.systems }}" ]; then
            echo "You must specify a value for systems. If you want to run all, specify ALL. If you want to run none, specify NONE."
            exit 1
          fi

    - name: Cleanup job workspace
      id: cleanup-job-workspace
      run: |
          rm -rf "${{ github.workspace }}"
          mkdir "${{ github.workspace }}"

    - name: Checkout code
      uses: actions/checkout@v4
      with:
        # spread uses tags as delta reference
        fetch-depth: 0

    - name: set SPREAD_SNAPD_DEB_FROM_REPO=1
      if: ${{ inputs.spread-snapd-deb-from-repo == 'true' }}
      run: echo "SPREAD_SNAPD_DEB_FROM_REPO=1" >> $GITHUB_ENV

    - name: Get previous attempt
      id: get-previous-attempt
      run: |
        echo "previous_attempt=$(( ${{ github.run_attempt }} - 1 ))" >> $GITHUB_OUTPUT
      shell: bash

    - name: Get previous cache
      uses: actions/cache@v4
      with:
        path: "${{ github.workspace }}/.test-results"
        key: "${{ github.job }}-results-${{ github.run_id }}-${{ inputs.group }}-${{ steps.get-previous-attempt.outputs.previous_attempt }}"

    - name: Prepare test results env and vars
      id: prepare-test-results-env
      run: |
          # Create test results directories and save vars
          TEST_RESULTS_DIR="${{ github.workspace }}/.test-results"
          echo "TEST_RESULTS_DIR=$TEST_RESULTS_DIR" >> $GITHUB_ENV

          # Save the var with the failed tests file
          echo "FAILED_TESTS_FILE=$TEST_RESULTS_DIR/failed-tests" >> $GITHUB_ENV

          # Make sure the test results dirs are created
          # This step has to be after the cache is restored
          mkdir -p "$TEST_RESULTS_DIR"

    - name: Prepare nested env vars
      if: contains(github.event.pull_request.labels.*.name, 'Run nested') && startsWith(inputs.group, 'nested-')
      run: |
          echo "RUN_NESTED=true" >> "$GITHUB_ENV"

    - name: Get changed files
      id: changed-files
      uses: tj-actions/changed-files@v45.0.6

    - name: Save changes files
      run: |
          CHANGED_FILES="${{ steps.changed-files.outputs.all_modified_files }}"
          echo "CHANGED_FILES=$CHANGED_FILES" >> $GITHUB_ENV
          echo "The changed files found are: $CHANGED_FILES"

    - name: Check failed tests to run
      if: "!contains(github.event.pull_request.labels.*.name, 'Run all')"
      run: |
          # Save previous failed test results in FAILED_TESTS env var
          FAILED_TESTS=""
          if [ -f "$FAILED_TESTS_FILE" ]; then
              echo "Failed tests file found"
              FAILED_TESTS="$(cat $FAILED_TESTS_FILE)"
              echo "Failed tests to run: $FAILED_TESTS"
              echo "FAILED_TESTS=$FAILED_TESTS" >> $GITHUB_ENV
          fi

    - name: Setup run tests variable
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
      run: |
          get_new_tasks() {
            local prefix=$1
            local changes_param=$2
            # The tests are just filtered when the change is a PR
            # When 'Run Nested' label is added in a PR, all the nested tests have to be executed
            TASKS_TO_RUN=""
            if [ -z "${{ github.event.number }}" ] || [ "$RUN_NESTED" = 'true' ] || [ -z "${{ inputs.rules }}" ] || [ -z "$changes_param" ]; then
                for TASKS in ${{ inputs.tasks }}; do
                    TASKS_TO_RUN="$TASKS_TO_RUN $prefix:$TASKS"
                done
            else
                NEW_TASKS="$(./tests/lib/external/snapd-testing-tools/utils/spread-filter -r ./tests/lib/spread/rules/${{ inputs.rules }}.yaml -p "$prefix" $changes_param)"
                TASKS_TO_RUN="$NEW_TASKS"
            fi
            echo "$TASKS_TO_RUN"
          }

          CHANGES_PARAM=""
          for CHANGE in $CHANGED_FILES; do
              CHANGES_PARAM="$CHANGES_PARAM -c $CHANGE"
          done
          RUN_TESTS=""
          # Save previous failed test results in FAILED_TESTS env var
          if [ -n "$FAILED_TESTS" ]; then
              RUN_TESTS="$FAILED_TESTS"
          elif [ "${{ inputs.systems }}" = "ALL" ]; then
              RUN_TESTS=$(get_new_tasks "${{ inputs.backend }}" "$CHANGES_PARAM")
          elif [ "${{ inputs.systems }}" != "NONE" ]; then
              for SYSTEM in ${{ inputs.systems }}; do
                NEW_TASKS="$(get_new_tasks "${{ inputs.backend }}:$SYSTEM" "$CHANGES_PARAM")"
                if [ -z "$RUN_TESTS" ]; then
                    RUN_TESTS="$NEW_TASKS"
                else
                    RUN_TESTS="$RUN_TESTS $NEW_TASKS"
                fi
              done
          fi
          echo RUN_TESTS="$RUN_TESTS"  >> $GITHUB_ENV

    - name: Check if system is required
      if: always()
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        required=$(gh api repos/${{ github.repository }}/rules/branches/master | \
          jq -r '.[] | select(.type=="required_status_checks") | .parameters.required_status_checks' | \
          grep -q "spread ${{ inputs.group }} /" && echo "req" || echo "not-req")
        echo REQUIRED="$required" >> $GITHUB_ENV

    - name: Setup grafana parameters
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
      run: |
          # Configure parameters to filter logs (these logs are sent read by grafana agent)
          CHANGE_ID="${{ github.event.number }}"
          if [ -z "$CHANGE_ID" ]; then
            CHANGE_ID="main"
          fi
          fundamental=""
          if [ "${{ inputs.is-fundamental }}" = 'true' ]; then
            fundamental="fund"
          fi
          FILTERED_LOG_FILE="spread_${CHANGE_ID}_${fundamental}_${REQUIRED}_n${{ github.run_attempt }}.filtered.log"
          # The log-filter tool is used to filter the spread logs to be stored
          echo FILTER_PARAMS="-o $FILTERED_LOG_FILE -e Debug -e WARNING: -f Failed=NO_LINES -f Error=NO_LINES"  >> $GITHUB_ENV
          echo FILTERED_LOG_FILE="$FILTERED_LOG_FILE"  >> $GITHUB_ENV

          # Add start line to filtered log
          echo "GRAFANA START: pr ${CHANGE_ID} attempt ${{ github.run_attempt }} run ${{ github.run_id }} group ${{ inputs.group }}" > "$FILTERED_LOG_FILE"

    - name: Download built snap (amd64)
      uses: actions/download-artifact@v4
      if: "${{ !inputs.use-snapd-snap-from-master && !contains(inputs.group, '-arm64') && !endsWith(inputs.group, '-fips') }}"
      with:
        name: snap-files-amd64-default-test
        # eg. snapd_1337.2.65.1+g97.d35b459_amd64.snap
        pattern: snapd_1337.*.snap
        path: "${{ github.workspace }}/built-snap"

    - name: Download built snap (arm64)
      if: "contains(inputs.group, '-arm64') && !endsWith(inputs.group, '-fips')"
      uses: actions/download-artifact@v4
      with:
        name: snap-files-arm64-default-test
        pattern: snapd_1337.*.snap
        # eg. snapd_1337.2.65.1+g97.d35b459_amd64.snap
        path: "${{ github.workspace }}/built-snap"

    - name: Download built FIPS snap (amd64)
      uses: actions/download-artifact@v4
      # eg. ubuntu-fips
      if: "${{ !inputs.use-snapd-snap-from-master && !contains(inputs.group, '-arm64') && endsWith(inputs.group, '-fips') }}"
      with:
        name: snap-files-amd64-FIPS-test
        # eg. snapd_1337.2.65.1+g97.d35b459-fips_amd64.snap
        pattern: snapd_1337.*-fips_*.snap
        path: "${{ github.workspace }}/built-snap"

    - name: Rename imported snap
      if: "${{ !inputs.use-snapd-snap-from-master }}"
      run: |
        for snap in built-snap/snapd_1337.*.snap; do
          mv -v "${snap}" "${snap}.keep"
        done

    - name: Run spread tests
      if: "!contains(github.event.pull_request.labels.*.name, 'Skip spread')"
      env:
          SPREAD_GOOGLE_KEY: ${{ secrets.SPREAD_GOOGLE_KEY }}
      run: |
          # Register a problem matcher to highlight spread failures
          echo "::add-matcher::.github/spread-problem-matcher.json"
          set -x
          SPREAD=spread
          if [[ "${{ inputs.group }}" =~ nested- ]]; then
            export NESTED_BUILD_SNAPD_FROM_CURRENT=true
            export NESTED_ENABLE_KVM=true
          fi

          export SPREAD_USE_PREBUILT_SNAPD_SNAP=true
          
          if [ "${{ inputs.use-snapd-snap-from-master }}" = true ]; then
            export SPREAD_USE_SNAPD_SNAP_URL=https://storage.googleapis.com/snapd-spread-tests/snapd-tests/snaps/snapd_master_amd64.snap
          fi

          # This could be the case when either there are not systems for a group or
          # the list of tests to run is empty
          if [ -z "$RUN_TESTS" ]; then
            echo "No tests to run, exiting..."
            exit 0
          fi

          # Add openstack backend definition to spread.yaml
          if [ "${{ inputs.backend }}" = openstack ]; then
              ./tests/lib/spread/add-backend tests/lib/spread/backend.openstack.yaml spread.yaml
          fi

          spread_list="$($SPREAD -list $RUN_TESTS 2>&1 || true)"

          if [[ "$spread_list" =~ amazon-linux-2023 ]]; then
              # Amazon Linux 2023 has no xdelta, however we cannot disable
              # xdelta on a per-target basis as it's used in the repack section
              # of spread.yaml, which is shared by all targets, so all systems
              # in this batch will not use delta for transferring project data
              echo "Disabling xdelta support"
              export NO_DELTA=1
          fi

          if grep -q "nothing matches provider filter" <<< "$spread_list"; then
            echo "No tests to run, exiting..."
            exit 0
          fi

          # Run spread tests
          # "pipefail" ensures that a non-zero status from the spread is
          # propagated; and we use a subshell as this option could trigger
          # undesired changes elsewhere
          echo "Running command: $SPREAD $RUN_TESTS"
          (
            set -o pipefail 
            $SPREAD -no-debug-output -logs spread-logs $RUN_TESTS | \
              ./tests/lib/external/snapd-testing-tools/utils/log-filter $FILTER_PARAMS | \
              tee spread.log
          )

    - name: Upload spread logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: "spread-logs-${{ inputs.group }}-${{ inputs.systems }}"
        path: "spread-logs/*.log"
        if-no-files-found: ignore

    - name: Discard spread workers
      if: always()
      run: |
        shopt -s nullglob;
        for r in .spread-reuse.*.yaml; do
          spread -discard -reuse-pid="$(echo "$r" | grep -o -E '[0-9]+')";
        done

    - name: Report spread errors
      if: always()
      run: |
        if [ -e spread.log ]; then
            echo "Running spread log analyzer"
            ACTIONS_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
            ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --output spread-results.json --cut 1 >/dev/null
            while IFS= read -r line; do
                if [ ! -z "$line" ]; then
                    echo "Adding failed test line to filtered log"
                    echo "GRAFANA FAILED: $line $ACTIONS_URL" | tee -a "$FILTERED_LOG_FILE"
                fi
            done <<< $(jq -r '.[] | select( .type == "info" ) | select( .info_type == "Error" ) | "\(.verb) \(.task)"' spread-results.json)
        else
            echo "No spread log found, skipping errors reporting"
        fi

    - name: Analyze spread test results
      if: always()
      run: |
          if [ -f spread.log ]; then
              echo "Running spread log parser"
              ./tests/lib/external/snapd-testing-tools/utils/log-parser spread.log --output spread-results.json

              # Add openstack backend definition to spread.yaml
              if [ "${{ inputs.backend }}" = openstack ]; then
                  ./tests/lib/spread/add-backend tests/lib/spread/backend.openstack.yaml spread.yaml
              fi

              echo "Running spread log analyzer"
              ./tests/lib/external/snapd-testing-tools/utils/log-analyzer list-reexecute-tasks "$RUN_TESTS" spread-results.json > "$FAILED_TESTS_FILE"

              echo "List of failed tests saved"
              cat "$FAILED_TESTS_FILE"
          else
              echo "No spread log found, saving empty list of failed tests"
              touch "$FAILED_TESTS_FILE"
          fi
          
          # Set the TEST_FAILED variable if there are any failed tests
          if [ -n "$(cat $FAILED_TESTS_FILE)" ]; then
            echo "TEST_FAILED=true" >> $GITHUB_ENV
          fi

    - name: Save spread test results to cache
      if: always()
      uses: actions/cache/save@v4
      with:
        path: "${{ github.workspace }}/.test-results"
        key: "${{ github.job }}-results-${{ github.run_id }}-${{ inputs.group }}-${{ github.run_attempt }}"

    - name: Save spread.json as an artifact
      if: ${{ failure() && github.event.pull_request.number && env.TEST_FAILED == 'true' }}
      uses: actions/upload-artifact@v4
      with:
        name: "spread-json-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.group }}" 
        path: "spread-results.json"
        if-no-files-found: ignore