soak.yml

# Soak test vector
#
# This workflow runs our 'soak' tests, which are relative evaluations of the
# base SHA for the PR to whatever SHA was just pushed into the project (unless
# that SHA happens to be master branch HEAD). The goal is to give quick-ish
# feedback on all-up vector for a variety of configs as to whether throughput
# performance has gone down, gotten more variable in the pushed SHA.
#
# Soaks are always done relative to the pushed SHA, meaning any changes you
# introduce to the soak tests will be picked up both for the base SHA soak and
# your current SHA. Tags are SHA-SHA. The first SHA is the one that triggered
# this workflow, the second is the one of the vector being tested. For
# comparison images the two SHAs are identical.
name: Soak

on:
  pull_request:
    paths-ignore:
      - "docs/**"
      - "rfcs/**"
      - "website/**"

jobs:
  cancel-previous:
    runs-on: ubuntu-20.04
    timeout-minutes: 3
    steps:
      - uses: styfle/cancel-workflow-action@0.9.1
        with:
          access_token: ${{ secrets.GITHUB_TOKEN }}
          all_but_latest: true # can cancel workflows scheduled later

  compute-soak-meta:
    name: Compute metadata for soaks
    runs-on: ubuntu-20.04
    outputs:
      pr-number: ${{ steps.pr-metadata.outputs.PR_NUMBER }}
      comparison-sha: ${{ steps.comparison.outputs.COMPARISON_SHA }}
      comparison-tag: ${{ steps.comparison.outputs.COMPARISON_TAG }}
      baseline-sha: ${{ steps.baseline.outputs.BASELINE_SHA }}
      baseline-tag: ${{ steps.baseline.outputs.BASELINE_TAG }}
      vector-cpus: ${{ steps.system.outputs.VECTOR_CPUS }}
      soak-cpus: ${{ steps.system.outputs.SOAK_CPUS }}
      soak-memory: ${{ steps.system.outputs.SOAK_MEMORY }}
      coefficient-of-variation: ${{ steps.system.outputs.COEFFICIENT_OF_VARIATION }}
      erratic-soaks: ${{ steps.system.outputs.ERRATIC_SOAKS }}
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.base_ref }}
          path: baseline-vector

      - name: Report on PR metadata
        id: pr-metadata
        run: |
          export PR_NUMBER=${{ github.event.number }}
          echo "::set-output name=PR_NUMBER::${PR_NUMBER}"
          echo "PR number: ${PR_NUMBER}"

      - name: Setup comparison variables
        id: comparison
        run: |
          export COMPARISON_SHA=${{ github.event.pull_request.head.sha }}
          export COMPARISON_TAG="${{ github.event.pull_request.head.sha }}-${{ github.event.pull_request.head.sha }}"

          echo "comparison sha is: ${COMPARISON_SHA}"
          echo "comparison tag is: ${COMPARISON_TAG}"

          echo "::set-output name=COMPARISON_TAG::${COMPARISON_TAG}"
          echo "::set-output name=COMPARISON_SHA::${COMPARISON_SHA}"

      - name: Setup baseline variables
        id: baseline
        run: |
          pushd baseline-vector
          export BASELINE_SHA=$(git rev-parse HEAD)
          popd

          export BASELINE_TAG="${{ github.event.pull_request.head.sha }}-${BASELINE_SHA}"
          echo "baseline sha is: ${BASELINE_SHA}"
          echo "baseline tag is: ${BASELINE_TAG}"

          echo "::set-output name=BASELINE_TAG::${BASELINE_TAG}"
          echo "::set-output name=BASELINE_SHA::${BASELINE_SHA}"

      - name: Setup system details
        id: system
        run: |
          export SOAK_CPUS="7"
          export SOAK_MEMORY="30g"
          export VECTOR_CPUS="4"
          export COEFFICIENT_OF_VARIATION="0.3"
          export ERRATIC_SOAKS="http_pipelines_blackhole,http_pipelines_blackhole_acks,http_to_http_acks,http_datadog_filter_blackhole"

          echo "soak cpus total: ${SOAK_CPUS}"
          echo "soak memory total: ${SOAK_MEMORY}"
          echo "vector cpus: ${VECTOR_CPUS}"
          echo "coefficient of variation limit: ${COEFFICIENT_OF_VARIATION}"
          echo "list of erratic soaks: ${ERRATIC_SOAKS}"

          echo "::set-output name=SOAK_CPUS::${SOAK_CPUS}"
          echo "::set-output name=COEFFICIENT_OF_VARIATION::${COEFFICIENT_OF_VARIATION}"
          echo "::set-output name=SOAK_MEMORY::${SOAK_MEMORY}"
          echo "::set-output name=VECTOR_CPUS::${VECTOR_CPUS}"
          echo "::set-output name=ERRATIC_SOAKS::${ERRATIC_SOAKS}"

  compute-test-plan:
    name: Compute soak test plan
    runs-on: ubuntu-20.04
    needs: [compute-soak-meta]
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3

      - uses: actions/github-script@v6
        id: set-matrix
        with:
          script: |
            const fs = require('fs')
            target = fs.readdirSync('soaks/tests') // read names of directories in soaks/tests into an array

            // Create the matrix that will be used as the strategy for soak test image builds and experiments
            //
            // Each field of this object has an array of elements that will be
            // cartesian producted across the others to generate the full list of
            // jobs to run. For example, the first job might have the values:
            // { target: "fluent_remap_aws_firehose", replica: [1] }
            const matrix = {
              target: target,     // run each experiment
            }

            // export this variable to be available to other github steps
            core.setOutput('matrix', matrix)
      - name: Dump matrix context
        env:
          MATRIX_CONTEXT: ${{ toJson(steps.set-matrix.outputs.matrix) }}
        run: echo "$MATRIX_CONTEXT"

  build-image-baseline:
    name: Build baseline 'soak-vector' container
    runs-on: [linux, soak, soak-builder]
    needs: [compute-soak-meta]
    steps:
      - uses: colpal/actions-clean@v1

      - uses: actions/checkout@v3

      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.compute-soak-meta.outputs.baseline-sha }}
          path: baseline-vector

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681
        with:
          flavor: |
            latest=false
            prefix=
            suffix=
          images: vector
          tags: type=raw, value=${{ needs.compute-soak-meta.outputs.baseline-tag }}

      - name: Build and push 'soak-vector' image
        uses: docker/build-push-action@v2.10.0
        with:
          context: baseline-vector/
          file: soaks/Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          outputs: type=docker,dest=/tmp/baseline-image.tar

      - name: Upload image as artifact
        uses: actions/upload-artifact@v2
        with:
          name: baseline-image
          path: /tmp/baseline-image.tar

  build-image-comparison:
    name: Build comparison 'soak-vector' container
    runs-on: [linux, soak, soak-builder]
    needs: [compute-soak-meta]
    steps:
      - uses: colpal/actions-clean@v1

      - uses: actions/checkout@v3

      - uses: actions/checkout@v3
        with:
          ref: ${{ needs.compute-soak-meta.outputs.comparison-sha }}
          path: comparison-vector

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681
        with:
          flavor: |
            latest=false
            prefix=
            suffix=
          images: vector
          tags: type=raw, value=${{ needs.compute-soak-meta.outputs.comparison-tag }}

      - name: Build 'soak-vector' image
        uses: docker/build-push-action@v2.10.0
        with:
          context: comparison-vector/
          file: soaks/Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          outputs: type=docker,dest=/tmp/comparison-image.tar

      - name: Upload image as artifact
        uses: actions/upload-artifact@v2
        with:
          name: comparison-image
          path: /tmp/comparison-image.tar

  soak-baseline:
    name: Soak (${{ matrix.target }}) - baseline
    runs-on: [self-hosted, linux, x64, soak]
    needs: [compute-soak-meta, compute-test-plan, build-image-baseline]
    strategy:
      matrix: ${{ fromJson(needs.compute-test-plan.outputs.matrix) }}
    steps:
      - uses: colpal/actions-clean@v1

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download baseline image
        uses: actions/download-artifact@v2
        with:
          name: baseline-image
          path: /tmp

      - name: Load baseline image
        run: |
          docker load --input /tmp/baseline-image.tar

      - name: Run baseline experiment
        run: |
          rm -rf /tmp/${{ github.run_id }}-${{ github.run_attempt }}/
          mkdir -p /tmp/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.target }}/
          ./soaks/bin/soak_one.sh --soak ${{ matrix.target }} \
                                  --build-image "false" \
                                  --variant "baseline" \
                                  --tag ${{ needs.compute-soak-meta.outputs.baseline-tag }} \
                                  --cpus ${{ needs.compute-soak-meta.outputs.soak-cpus }} \
                                  --memory ${{ needs.compute-soak-meta.outputs.soak-memory }} \
                                  --vector-cpus ${{ needs.compute-soak-meta.outputs.vector-cpus }} \
                                  --replicas 4 \
                                  --capture-dir /tmp/${{ github.run_id }}-${{ github.run_attempt }}

      - name: Upload timing captures
        uses: actions/upload-artifact@v1
        with:
          name: ${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.target }}-baseline
          path: /tmp/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.target }}/

      - name: Clear up unused images
        run: |
          minikube delete --all --purge
          docker system prune --all --volumes --force

  detect-erratic-baseline:
    name: Erratic detection - baseline
    runs-on: ubuntu-20.04
    needs:
      - compute-soak-meta
      - soak-baseline

    steps:
      - name: Set up Python3
        uses: actions/setup-python@v3
        with:
          python-version: "3.10.2"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install scipy==1.8.* pandas==1.4.* tabulate==0.8.*

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download captures artifact
        uses: actions/download-artifact@v2
        with:
          path: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Display structure of downloaded files
        run: ls -R
        working-directory: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Detect erratic
        run: |
          ./soaks/bin/detect_erratic --capture-dir ${{ github.run_id }}-${{ github.run_attempt }}-captures/ \
                                     --vector-cpus ${{ needs.compute-soak-meta.outputs.vector-cpus }} \
                                     --warmup-seconds 30 \
                                     --variant baseline \
                                     --coefficient-of-variation-limit ${{ needs.compute-soak-meta.outputs.coefficient-of-variation }} \
                                     --erratic-soaks ${{ needs.compute-soak-meta.outputs.erratic-soaks }}

  soak-comparison:
    name: Soak (${{ matrix.target }}) - comparison
    runs-on: [self-hosted, linux, x64, soak]
    needs: [compute-soak-meta, compute-test-plan, build-image-comparison]
    strategy:
      matrix: ${{ fromJson(needs.compute-test-plan.outputs.matrix) }}
    steps:
      - uses: colpal/actions-clean@v1

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download comparison image
        uses: actions/download-artifact@v2
        with:
          name: comparison-image
          path: /tmp

      - name: Load comparison image
        run: |
          docker load --input /tmp/comparison-image.tar

      - name: Run comparison experiment
        run: |
          rm -rf /tmp/${{ github.run_id }}-${{ github.run_attempt }}/
          mkdir -p /tmp/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.target }}/
          ./soaks/bin/soak_one.sh --soak ${{ matrix.target }} \
                                  --build-image "false" \
                                  --variant "comparison" \
                                  --tag ${{ needs.compute-soak-meta.outputs.comparison-tag }} \
                                  --cpus ${{ needs.compute-soak-meta.outputs.soak-cpus }} \
                                  --memory ${{ needs.compute-soak-meta.outputs.soak-memory }} \
                                  --vector-cpus ${{ needs.compute-soak-meta.outputs.vector-cpus }} \
                                  --replicas 4 \
                                  --capture-dir /tmp/${{ github.run_id }}-${{ github.run_attempt }}

      - name: Upload timing captures
        uses: actions/upload-artifact@v1
        with:
          name: ${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.target }}-comparison
          path: /tmp/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.target }}/

      - name: Clear up unused images
        run: |
          minikube delete --all --purge
          docker system prune --all --volumes --force

  detect-erratic-comparison:
    name: Erratic detection - comparison
    runs-on: ubuntu-20.04
    needs:
      - compute-soak-meta
      - soak-comparison

    steps:
      - name: Set up Python3
        uses: actions/setup-python@v3
        with:
          python-version: "3.10.2"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install scipy==1.8.* pandas==1.4.* tabulate==0.8.*

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download captures artifact
        uses: actions/download-artifact@v2
        with:
          path: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Display structure of downloaded files
        run: ls -R
        working-directory: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Detect erratic
        run: |
          ./soaks/bin/detect_erratic --capture-dir ${{ github.run_id }}-${{ github.run_attempt }}-captures/ \
                                     --vector-cpus ${{ needs.compute-soak-meta.outputs.vector-cpus }} \
                                     --warmup-seconds 30 \
                                     --variant comparison \
                                     --coefficient-of-variation-limit ${{ needs.compute-soak-meta.outputs.coefficient-of-variation }} \
                                     --erratic-soaks ${{ needs.compute-soak-meta.outputs.erratic-soaks }}

  analyze-results:
    name: Soak analysis
    runs-on: ubuntu-20.04
    needs:
      - compute-soak-meta
      - soak-baseline
      - soak-comparison

    steps:
      - name: Set up Python3
        uses: actions/setup-python@v3
        with:
          python-version: "3.10.2"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install scipy==1.8.* pandas==1.4.* tabulate==0.8.*

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download captures artifact
        uses: actions/download-artifact@v2
        with:
          path: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Display structure of downloaded files
        run: ls -R
        working-directory: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Analyze captures
        run: |
          ./soaks/bin/analyze_experiment --capture-dir ${{ github.run_id }}-${{ github.run_attempt }}-captures/ \
                                         --baseline-sha ${{ needs.compute-soak-meta.outputs.baseline-sha }} \
                                         --comparison-sha ${{ needs.compute-soak-meta.outputs.comparison-sha }} \
                                         --vector-cpus ${{ needs.compute-soak-meta.outputs.vector-cpus }} \
                                         --warmup-seconds 30 \
                                         --coefficient-of-variation-limit ${{ needs.compute-soak-meta.outputs.coefficient-of-variation }} \
                                         --erratic-soaks ${{ needs.compute-soak-meta.outputs.erratic-soaks }} \
                                         --mean-drift-percentage 7 \
                                         --p-value 0.1 > /tmp/${{ github.run_id}}-${{ github.run_attempt }}-analysis
      - uses: actions/upload-artifact@v2
        with:
          name: soak-analysis
          path: /tmp/${{ github.run_id }}-${{ github.run_attempt }}-analysis

      - name: Save PR number for subsequent workflow
        run: |
          echo ${{ github.event.number }} > /tmp/pr.txt
      - uses: actions/upload-artifact@v2
        with:
          name: pr-number
          path: /tmp/pr.txt

  detect-regressions:
    name: Regression analysis
    runs-on: ubuntu-20.04
    needs:
      - compute-soak-meta
      - soak-baseline
      - soak-comparison

    steps:
      - name: Set up Python3
        uses: actions/setup-python@v3
        with:
          python-version: "3.10.2"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install scipy==1.8.* pandas==1.4.* tabulate==0.8.*

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download captures artifact
        uses: actions/download-artifact@v2
        with:
          path: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Display structure of downloaded files
        run: ls -R
        working-directory: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Detect regressions
        run: |
          ./soaks/bin/detect_regressions --capture-dir ${{ github.run_id }}-${{ github.run_attempt }}-captures/ \
                                         --warmup-seconds 30 \
                                         --erratic-soaks ${{ needs.compute-soak-meta.outputs.erratic-soaks }} \
                                         --mean-drift-percentage 7 \
                                         --p-value 0.1

  plot-analysis:
    name: Plot analysis
    runs-on: ubuntu-20.04
    needs:
      - compute-soak-meta
      - soak-baseline
      - soak-comparison

    steps:
      - name: Set up Python3
        uses: actions/setup-python@v3
        with:
          python-version: "3.10.2"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install scipy==1.8.* pandas==1.4.* seaborn==0.11.* tabulate==0.8.*

      - name: Check out the repo
        uses: actions/checkout@v3

      - name: Download captures artifact
        uses: actions/download-artifact@v2
        with:
          path: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Display structure of downloaded files
        run: ls -R
        working-directory: ${{ github.run_id }}-${{ github.run_attempt }}-captures/

      - name: Plot analysis
        run: |
          mkdir -p ${{ github.run_id }}-${{ github.run_attempt }}-captures/plots/
          ./soaks/bin/plot_analysis --capture-dir ${{ github.run_id }}-${{ github.run_attempt }}-captures/ \
                                    --output-dir ${{ github.run_id }}-${{ github.run_attempt }}-captures/plots/ \
                                    --vector-cpus ${{ needs.compute-soak-meta.outputs.vector-cpus }}

      - name: Upload plots
        uses: actions/upload-artifact@v1
        with:
          name: ${{ github.run_id }}-${{ github.run_attempt }}-captures-plots
          path: "${{ github.run_id }}-${{ github.run_attempt }}-captures/plots"