diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml new file mode 100644 index 0000000000000..449f4210b624a --- /dev/null +++ b/.github/workflows/changes.yml @@ -0,0 +1,333 @@ +# This workflow identifies changes between the base and the head ref, for use in +# other workflows to decide if they should be executed. + +name: Identify Changes + +on: + workflow_call: + # These inputs allow the filter action to be able to access the correct refs for + # comparison in changes detection, it is required as this is called from the + # merge_group context. + inputs: + base_ref: + required: true + type: string + head_ref: + required: true + type: string + outputs: + source: + value: ${{ jobs.changes.outputs.source }} + dependencies: + value: ${{ jobs.changes.outputs.dependencies }} + internal_events: + value: ${{ jobs.changes.outputs.internal_events }} + cue: + value: ${{ jobs.changes.outputs.cue }} + component_docs: + value: ${{ jobs.changes.outputs.component_docs }} + markdown: + value: ${{ jobs.changes.outputs.markdown }} + install: + value: ${{ jobs.changes.outputs.install }} + k8s: + value: ${{ jobs.changes.outputs.k8s }} + all-int: + value: ${{ jobs.changes.outputs.all-int }} + amqp: + value: ${{ jobs.changes.outputs.amqp }} + appsignal: + value: ${{ jobs.changes.outputs.appsignal }} + aws: + value: ${{ jobs.changes.outputs.aws }} + axiom: + value: ${{ jobs.changes.outputs.axiom }} + azure: + value: ${{ jobs.changes.outputs.azure }} + clickhouse: + value: ${{ jobs.changes.outputs.clickhouse }} + databend: + value: ${{ jobs.changes.outputs.databend }} + datadog: + value: ${{ jobs.changes.outputs.datadog }} + dnstap: + value: ${{ jobs.changes.outputs.dnstap }} + docker-logs: + value: ${{ jobs.changes.outputs.docker-logs }} + elasticsearch: + value: ${{ jobs.changes.outputs.elasticsearch }} + eventstoredb: + value: ${{ jobs.changes.outputs.eventstoredb }} + fluent: + value: ${{ jobs.changes.outputs.fluent }} + gcp: + value: ${{ jobs.changes.outputs.gcp }} + humio: + value: ${{ jobs.changes.outputs.humio }} + http-client: + value: ${{ jobs.changes.outputs.http-client }} + influxdb: + value: ${{ jobs.changes.outputs.influxdb }} + kafka: + value: ${{ jobs.changes.outputs.kafka }} + logstash: + value: ${{ jobs.changes.outputs.logstash }} + loki: + value: ${{ jobs.changes.outputs.loki }} + mongodb: + value: ${{ jobs.changes.outputs.mongodb }} + nats: + value: ${{ jobs.changes.outputs.nats }} + nginx: + value: ${{ jobs.changes.outputs.nginx }} + opentelemetry: + value: ${{ jobs.changes.outputs.opentelemetry }} + postgres: + value: ${{ jobs.changes.outputs.postgres }} + prometheus: + value: ${{ jobs.changes.outputs.prometheus }} + pulsar: + value: ${{ jobs.changes.outputs.pulsar }} + redis: + value: ${{ jobs.changes.outputs.redis }} + splunk: + value: ${{ jobs.changes.outputs.splunk }} + webhdfs: + value: ${{ jobs.changes.outputs.webhdfs }} + +jobs: + changes: + runs-on: ubuntu-20.04 + # Set job outputs to values from filter step + outputs: + # General source code + source: ${{ steps.filter.outputs.source }} + dependencies: ${{ steps.filter.outputs.dependencies }} + internal_events: ${{ steps.filter.outputs.internal_events }} + cue: ${{ steps.filter.outputs.cue }} + component_docs: ${{ steps.filter.outputs.component_docs }} + markdown: ${{ steps.filter.outputs.markdown }} + install: ${{ steps.filter.outputs.install }} + # K8s + k8s: ${{ steps.filter.outputs.k8s }} + # Integrations + all-int: ${{ steps.filter.outputs.all-int }} + amqp: ${{ steps.filter.outputs.amqp }} + appsignal: ${{ steps.filter.outputs.appsignal}} + aws: ${{ steps.filter.outputs.aws }} + axiom: ${{ steps.filter.outputs.axiom }} + azure: ${{ steps.filter.outputs.azure }} + clickhouse: ${{ steps.filter.outputs.clickhouse }} + databend: ${{ steps.filter.outputs.databend }} + datadog: ${{ steps.filter.outputs.datadog }} + dnstap: ${{ steps.filter.outputs.dnstap }} + docker-logs: ${{ steps.filter.outputs.docker-logs }} + elasticsearch: ${{ steps.filter.outputs.elasticsearch }} + eventstoredb: ${{ steps.filter.outputs.eventstoredb }} + fluent: ${{ steps.filter.outputs.fluent }} + gcp: ${{ steps.filter.outputs.gcp }} + humio: ${{ steps.filter.outputs.humio }} + http-client: ${{ steps.filter.outputs.http-client }} + influxdb: ${{ steps.filter.outputs.influxdb }} + kafka: ${{ steps.filter.outputs.kafka }} + logstash: ${{ steps.filter.outputs.logstash }} + loki: ${{ steps.filter.outputs.loki }} + mongodb: ${{ steps.filter.outputs.mongodb }} + nats: ${{ steps.filter.outputs.nats }} + nginx: ${{ steps.filter.outputs.nginx }} + opentelemetry: ${{ steps.filter.outputs.opentelemetry }} + postgres: ${{ steps.filter.outputs.postgres }} + prometheus: ${{ steps.filter.outputs.prometheus }} + pulsar: ${{ steps.filter.outputs.pulsar }} + redis: ${{ steps.filter.outputs.redis }} + splunk: ${{ steps.filter.outputs.splunk }} + webhdfs: ${{ steps.filter.outputs.webhdfs }} + steps: + - uses: actions/checkout@v3 + + - uses: dorny/paths-filter@v2 + id: filter + with: + base: ${{ inputs.base_ref }} + ref: ${{ inputs.head_ref }} + filters: | + source: + - ".github/workflows/test.yml" + - ".cargo/**" + - "benches/**" + - "lib/**" + - "proto/**" + - "scripts/**" + - "src/**" + - "tests/**" + - "build.rs" + - "Cargo.lock" + - "Cargo.toml" + - "Makefile" + - "rust-toolchain.toml" + - "vdev/**" + deny: + - 'deny.toml' + - "vdev/**" + dependencies: + - ".cargo/**" + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - '.github/workflows/pr.yml' + - 'Makefile' + - 'scripts/cross/**' + - "vdev/**" + cue: + - 'website/cue/**' + - "vdev" + component_docs: + - 'scripts/generate-component-docs.rb' + - "vdev/**" + markdown: + - '**/**.md' + - "vdev/**" + internal_events: + - 'src/internal_events/**' + - "vdev/**" + docker: + - 'distribution/docker/**' + - "vdev/**" + install: + - ".github/workflows/install-sh.yml" + - "distribution/install.sh" + k8s: + - "src/sources/kubernetes_logs/**" + all-int: + - "lib/vector-core/**" + amqp: + - "src/amqp.rs" + - "src/internal_events/amqp.rs" + - "src/sinks/amqp/**" + - "src/sources/amqp.rs" + - "src/sources/util/**" + - "src/sinks/util/**" + appsignal: + - "src/sinks/appsignal/**" + - "src/sinks/util/**" + aws: + - "src/aws_**" + - "src/internal_events/aws_**" + - "src/sources/aws_**" + - "src/sources/util/**" + - "src/sinks/aws_**" + - "src/sinks/util/**" + - "src/transforms/aws_**" + axiom: + - "src/sinks/axiom.rs" + - "src/sinks/util/**" + azure: + - "src/sinks/azure_**" + - "src/sinks/util/**" + clickhouse: + - "src/sinks/clickhouse/**" + - "src/sinks/util/**" + databend: + - "src/sinks/databend/**" + - "src/sinks/util/**" + datadog: + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sources/datadog_agent/**" + - "src/sinks/datadog/**" + - "src/sinks/datadog_archives.rs" + - "src/sinks/util/**" + docker-logs: + - "src/docker.rs" + - "src/internal_events/docker_logs.rs" + - "src/sources/docker_logs/**" + - "src/sources/util/**" + elasticsearch: + - "src/sinks/elasticsearch/**" + - "src/sinks/util/**" + eventstoredb: + - "src/internal_events/eventstoredb_metrics.rs" + - "src/sources/eventstoredb_metrics/**" + - "src/sources/util/**" + fluent: + - "src/internal_events/fluent.rs" + - "src/sources/fluent/**" + - "src/sources/util/**" + gcp: + - "src/internal_events/gcp_pubsub.rs" + - "src/sources/gcp_pubsub.rs" + - "src/sources/util/**" + - "src/sinks/gcp/**" + - "src/sinks/util/**" + - "src/gcp.rs" + humio: + - "src/sinks/humio/**" + - "src/sinks/util/**" + http-client: + - "src/sinks/http-client/**" + influxdb: + - "src/internal_events/influxdb.rs" + - "src/sinks/influxdb/**" + - "src/sinks/util/**" + kafka: + - "src/internal_events/kafka.rs" + - "src/sinks/kafka/**" + - "src/sinks/util/**" + - "src/sources/kafka.rs" + - "src/sources/util/**" + - "src/kafka.rs" + logstash: + - "src/sources/logstash.rs" + - "src/sources/util/**" + loki: + - "src/internal_events/loki.rs" + - "src/sinks/loki/**" + - "src/sinks/util/**" + mongodb: + - "src/internal_events/mongodb_metrics.rs" + - "src/sources/mongodb_metrics/**" + - "src/sources/util/**" + nats: + - "src/internal_events/nats.rs" + - "src/sources/nats.rs" + - "src/sources/util/**" + - "src/sinks/nats.rs" + - "src/sinks/util/**" + - "src/nats.rs" + nginx: + - "src/internal_events/nginx_metrics.rs" + - "src/sources/nginx_metrics/**" + - "src/sources/util/**" + opentelemetry: + - "src/sources/opentelemetry/**" + - "src/sources/util/**" + postgres: + - "src/internal_events/postgresql_metrics.rs" + - "src/sources/postgresql_metrics.rs" + - "src/sources/util/**" + prometheus: + - "src/internal_events/prometheus.rs" + - "src/sources/prometheus/**" + - "src/sources/util/**" + - "src/sinks/prometheus/**" + - "src/sinks/util/**" + pulsar: + - "src/internal_events/pulsar.rs" + - "src/sinks/pulsar/**" + - "src/sinks/util/**" + redis: + - "src/internal_events/redis.rs" + - "src/sources/redis/**" + - "src/sources/util/**" + - "src/sinks/redis.rs" + - "src/sinks/util/**" + splunk: + - "src/internal_events/splunk_hec.rs" + - "src/sources/splunk_hec/**" + - "src/sources/util/**" + - "src/sinks/splunk_hec/**" + - "src/sinks/util/**" + webhdfs: + - "src/sinks/webhdfs/**" + - "src/sinks/util/**" + diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml new file mode 100644 index 0000000000000..8ec1cf0ba8cc5 --- /dev/null +++ b/.github/workflows/cli.yml @@ -0,0 +1,63 @@ +name: CLI - Linux + +on: + workflow_call: + +jobs: + test-cli: + runs-on: [linux, ubuntu-20.04-8core] + env: + CARGO_INCREMENTAL: 0 + steps: + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: CLI - Linux + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - name: Cache Cargo registry + index + uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: bash scripts/environment/prepare.sh + - run: echo "::add-matcher::.github/matchers/rust.json" + - run: make test-cli + - name: Upload test results + run: scripts/upload-test-results.sh + if: always() + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@1.1.6 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: CLI - Linux + status: ${{ job.status }} diff --git a/.github/workflows/comment-trigger.yml b/.github/workflows/comment-trigger.yml new file mode 100644 index 0000000000000..e9c2c234233bb --- /dev/null +++ b/.github/workflows/comment-trigger.yml @@ -0,0 +1,118 @@ +# Comment Trigger +# +# This workflow is a central point for triggering workflow runs that normally run only as part of the merge queue, +# on demand by a comment. The exception being the integration tests, which have their own workflow file for +# comment triggers as the logic is a bit more complex. +# +# The available triggers are: +# +# /ci-run-all : runs all of the below +# /ci-run-cli : runs CLI - Linux +# /ci-run-misc : runs Miscellaneous - Linux +# /ci-run-component-features : runs Component Features - Linux +# /ci-run-cross : runs Cross +# /ci-run-unit-mac : runs Unit - Mac +# /ci-run-unit-windows : runs Unit - Windows +# /ci-run-environment : runs Environment Suite +# /ci-run-install : runs Update install.sh Suite +# /ci-run-regression : runs Regression Detection Suite + +name: Comment Trigger + +on: + issue_comment: + types: [created] + +env: + DD_ENV: "ci" + RUST_BACKTRACE: full + TEST_LOG: vector=debug + VERBOSE: true + CI: true + PROFILE: debug + # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps + # can be removed when we switch back to the upstream openssl-sys crate + CARGO_NET_GIT_FETCH_WITH_CLI: true + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue_comment.issue.id }}-${{ github.event.comment.body }} + cancel-in-progress: true + +jobs: + validate: + name: Validate comment + runs-on: ubuntu-latest + if: | + github.event.issue.pull_request && ( contains(github.event.comment.body, '/ci-run-all') + || contains(github.event.comment.body, '/ci-run-cli') + || contains(github.event.comment.body, '/ci-run-misc') + || contains(github.event.comment.body, '/ci-run-component-features') + || contains(github.event.comment.body, '/ci-run-cross') + || contains(github.event.comment.body, '/ci-run-unit-mac') + || contains(github.event.comment.body, '/ci-run-unit-windows') + || contains(github.event.comment.body, '/ci-run-environment') + || contains(github.event.comment.body, '/ci-run-install') + || contains(github.event.comment.body, '/ci-run-regression') + ) + steps: + - name: Validate issue comment + id: comment + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + cli: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-cli') + uses: ./.github/workflows/cli.yml + secrets: inherit + + misc: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-misc') + uses: ./.github/workflows/misc.yml + secrets: inherit + + component-features: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-component-features') + uses: ./.github/workflows/component_features.yml + secrets: inherit + + cross: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-cross') + uses: ./.github/workflows/cross.yml + secrets: inherit + + unit-mac: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-unit-mac') + uses: ./.github/workflows/unit_mac.yml + secrets: inherit + + unit-windows: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-unit-windows') + uses: ./.github/workflows/unit_windows.yml + secrets: inherit + + environment: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-environment') + uses: ./.github/workflows/environment.yml + secrets: inherit + + install: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-install') + uses: ./.github/workflows/install-sh.yml + secrets: inherit + + regression: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-regression') + uses: ./.github/workflows/regression.yml + secrets: inherit diff --git a/.github/workflows/baseline.yml b/.github/workflows/compilation-timings.yml similarity index 97% rename from .github/workflows/baseline.yml rename to .github/workflows/compilation-timings.yml index 2bc12be8c05c3..e96bea65ea946 100644 --- a/.github/workflows/baseline.yml +++ b/.github/workflows/compilation-timings.yml @@ -1,8 +1,8 @@ -# Executes various builds of vector to time the results in order to track build times. +# Executes various builds of vector to time the results in order to track compilation times. # # This workflow is unrelated to the Regression workflow. -name: Baseline Timings +name: Compilation Timings on: workflow_dispatch: diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml new file mode 100644 index 0000000000000..747483c1316cd --- /dev/null +++ b/.github/workflows/component_features.yml @@ -0,0 +1,46 @@ +name: Component Features - Linux + +on: + workflow_call: + +jobs: + check-component-features: + runs-on: [linux, ubuntu-20.04-8core] + steps: + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + uses: myrotvorets/set-commit-status-action@1.1.6 + if: ${{ github.event_name == 'issue_comment' }} + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Component Features - Linux + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: bash scripts/environment/prepare.sh + - run: echo "::add-matcher::.github/matchers/rust.json" + - run: make check-component-features + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@1.1.6 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Component Features - Linux + status: ${{ job.status }} diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml new file mode 100644 index 0000000000000..e5fcdceb84ffb --- /dev/null +++ b/.github/workflows/cross.yml @@ -0,0 +1,102 @@ +name: Cross + +on: + workflow_call: + +jobs: + cross-linux: + name: Cross - ${{ matrix.target }} + runs-on: [linux, ubuntu-20.04-8core] + env: + CARGO_INCREMENTAL: 0 + strategy: + matrix: + target: + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-gnu + - aarch64-unknown-linux-musl + - armv7-unknown-linux-gnueabihf + - armv7-unknown-linux-musleabihf + steps: + + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Cross + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + name: Cache Cargo registry + index + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - run: echo "::add-matcher::.github/matchers/rust.json" + - run: 'cargo install cross --version 0.2.4 --force --locked' + # Why is this build, not check? Because we need to make sure the linking phase works. + # aarch64 and musl in particular are notoriously hard to link. + # While it may be tempting to slot a `check` in here for quickness, please don't. + - run: make cross-build-${{ matrix.target }} + - uses: actions/upload-artifact@v3 + with: + name: "vector-debug-${{ matrix.target }}" + path: "./target/${{ matrix.target }}/debug/vector" + + - name: (PR comment) Set latest commit status as failed + uses: myrotvorets/set-commit-status-action@1.1.6 + if: failure() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Cross + status: 'failure' + + update-pr-status: + name: (PR comment) Signal result to PR + runs-on: ubuntu-20.04 + needs: cross-linux + if: needs.cross-linux.result == 'success' && github.event_name == 'issue_comment' + steps: + - name: Validate issue comment + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + - name: (PR comment) Get PR branch + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Submit PR result as success + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Cross + status: 'success' diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 2e08442f4eeef..e8c64e4fca5ab 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -1,11 +1,11 @@ name: Environment Suite on: - pull_request: {} + workflow_call: + workflow_dispatch: push: branches: - master - workflow_dispatch: env: VERBOSE: true @@ -15,8 +15,30 @@ jobs: publish-new-environment: runs-on: ubuntu-20.04 steps: - - name: Checkout + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Environment Suite + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + - name: Set up QEMU uses: docker/setup-qemu-action@v2.1.0 - name: Set up Docker Buildx @@ -48,3 +70,12 @@ jobs: push: ${{ github.ref == 'refs/heads/master' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@1.1.6 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Environment Suite + status: ${{ job.status }} diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index f9619dba9e542..c19ea3e9310a8 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -1,46 +1,71 @@ name: Update install.sh Suite on: - push: - branches: - - master - paths: - - '.github/workflows/install-sh.yml' - - 'distribution/install.sh' + workflow_call: workflow_dispatch: - jobs: + sync-install: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Update install.sh Suite + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + - run: pip3 install awscli --upgrade --user - env: AWS_ACCESS_KEY_ID: "${{ secrets.CI_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}" run: make sync-install + - name: (PR comment) Set latest commit status as failed + uses: myrotvorets/set-commit-status-action@1.1.6 + if: failure() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Update install.sh Suite + status: 'failure' + test-install: - needs: - - sync-install + needs: sync-install runs-on: ubuntu-20.04 steps: - run: sudo apt-get install --yes curl bc - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.vector.dev | bash -s -- -y - run: ~/.vector/bin/vector --version - install-shell-failure: - name: install-shell-failure - if: failure() - needs: - - sync-install - - test-install - runs-on: ubuntu-20.04 - steps: - - name: Discord notification - env: - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - uses: Ilshidur/action-discord@0.3.2 - with: - args: "Update of sh.vector.dev failed: " + - name: (PR comment) Get PR branch + if: github.event_name == 'issue_comment' + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as ${{ job.status }} + if: github.event_name == 'issue_comment' + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Update install.sh Suite + status: ${{ job.status }} diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml new file mode 100644 index 0000000000000..6481699d569f8 --- /dev/null +++ b/.github/workflows/integration-comment.yml @@ -0,0 +1,175 @@ +# Integration Test Comment +# +# This workflow runs one or more integration tests triggered by a comment in a PR. +# The comment include '/ci-run-integration'. +# Then, any if the integration names will trigger that specific integration. +# 'all' will trigger every integration to run. +# The order does not matter and can be anywhere inside the comment body. +# +# Examples: +# +# 1. Run a single integration test: +# +# /ci-run-integration amqp +# +# 2. Run three specific integration tests: +# +# /ci-run-integration dnstap redis amqp +# +# 3. Run all integration tests: +# +# /ci-run-integration all + +name: Integration Test Comment + +on: + issue_comment: + types: [created] + +env: + AWS_ACCESS_KEY_ID: "dummy" + AWS_SECRET_ACCESS_KEY: "dummy" + AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} + TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} + CONTAINER_TOOL: "docker" + DD_ENV: "ci" + DD_API_KEY: ${{ secrets.DD_API_KEY }} + RUST_BACKTRACE: full + TEST_LOG: vector=debug + VERBOSE: true + CI: true + PROFILE: debug + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.id }} + cancel-in-progress: true + +jobs: + prep-pr: + name: (PR comment) Signal pending to PR + runs-on: ubuntu-latest + if: contains(github.event.comment.body, '/ci-run-integration') || contains(github.event.comment.body, '/ci-run-all') + steps: + - name: Validate issue comment + if: github.event_name == 'issue_comment' + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + - name: (PR comment) Get PR branch + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: pending + + test-integration: + uses: ./.github/workflows/integration-test.yml + with: + if: ${{ matrix.run.if }} + test_name: ${{ matrix.run.test_name }} + needs: prep-pr + secrets: inherit + strategy: + fail-fast: false + matrix: + run: + - test_name: 'amqp' + if: ${{ contains(github.event.comment.body, 'amqp') || contains(github.event.comment.body, 'all') }} + - test_name: 'appsignal' + if: ${{ contains(github.event.comment.body, 'appsignal') || contains(github.event.comment.body, 'all') }} + - test_name: 'aws' + if: ${{ contains(github.event.comment.body, 'aws') || contains(github.event.comment.body, 'all') }} + - test_name: 'axiom' + if: ${{ contains(github.event.comment.body, 'axiom') || contains(github.event.comment.body, 'all') }} + - test_name: 'azure' + if: ${{ contains(github.event.comment.body, 'azure') || contains(github.event.comment.body, 'all') }} + - test_name: 'clickhouse' + if: ${{ contains(github.event.comment.body, 'clickhouse') || contains(github.event.comment.body, 'all') }} + - test_name: 'databend' + if: ${{ contains(github.event.comment.body, 'databend') || contains(github.event.comment.body, 'all') }} + - test_name: 'datadog-agent' + if: ${{ contains(github.event.comment.body, 'datadog') || contains(github.event.comment.body, 'all') }} + - test_name: 'datadog-logs' + if: ${{ contains(github.event.comment.body, 'datadog') || contains(github.event.comment.body, 'all') }} + - test_name: 'datadog-metrics' + if: ${{ contains(github.event.comment.body, 'datadog') || contains(github.event.comment.body, 'all') }} + - test_name: 'datadog-traces' + if: ${{ contains(github.event.comment.body, 'datadog') || contains(github.event.comment.body, 'all') }} + - test_name: 'dnstap' + if: ${{ contains(github.event.comment.body, 'dnstap') || contains(github.event.comment.body, 'all') }} + - test_name: 'docker-logs' + if: ${{ contains(github.event.comment.body, 'docker-logs') || contains(github.event.comment.body, 'all') }} + - test_name: 'elasticsearch' + if: ${{ contains(github.event.comment.body, 'elasticsearch') || contains(github.event.comment.body, 'all') }} + - test_name: 'eventstoredb' + if: ${{ contains(github.event.comment.body, 'eventstoredb') || contains(github.event.comment.body, 'all') }} + - test_name: 'fluent' + if: ${{ contains(github.event.comment.body, 'fluent') || contains(github.event.comment.body, 'all') }} + - test_name: 'gcp' + if: ${{ contains(github.event.comment.body, 'gcp') || contains(github.event.comment.body, 'all') }} + - test_name: 'humio' + if: ${{ contains(github.event.comment.body, 'humio') || contains(github.event.comment.body, 'all') }} + - test_name: 'http-client' + if: ${{ contains(github.event.comment.body, 'http-client') || contains(github.event.comment.body, 'all') }} + - test_name: 'influxdb' + if: ${{ contains(github.event.comment.body, 'influxdb') || contains(github.event.comment.body, 'all') }} + - test_name: 'kafka' + if: ${{ contains(github.event.comment.body, 'kafka') || contains(github.event.comment.body, 'all') }} + - test_name: 'logstash' + if: ${{ contains(github.event.comment.body, 'logstash') || contains(github.event.comment.body, 'all') }} + - test_name: 'loki' + if: ${{ contains(github.event.comment.body, 'loki') || contains(github.event.comment.body, 'all') }} + - test_name: 'mongodb' + if: ${{ contains(github.event.comment.body, 'mongodb') || contains(github.event.comment.body, 'all') }} + - test_name: 'nats' + if: ${{ contains(github.event.comment.body, 'nats') || contains(github.event.comment.body, 'all') }} + - test_name: 'nginx' + if: ${{ contains(github.event.comment.body, 'nginx') || contains(github.event.comment.body, 'all') }} + - test_name: 'opentelemetry' + if: ${{ contains(github.event.comment.body, 'opentelemetry') || contains(github.event.comment.body, 'all') }} + - test_name: 'postgres' + if: ${{ contains(github.event.comment.body, 'postgres') || contains(github.event.comment.body, 'all') }} + - test_name: 'prometheus' + if: ${{ contains(github.event.comment.body, 'prometheus') || contains(github.event.comment.body, 'all') }} + - test_name: 'pulsar' + if: ${{ contains(github.event.comment.body, 'pulsar') || contains(github.event.comment.body, 'all') }} + - test_name: 'redis' + if: ${{ contains(github.event.comment.body, 'redis') || contains(github.event.comment.body, 'all') }} + - test_name: 'shutdown' + if: ${{ contains(github.event.comment.body, 'shutdown') || contains(github.event.comment.body, 'all') }} + - test_name: 'splunk' + if: ${{ contains(github.event.comment.body, 'splunk') || contains(github.event.comment.body, 'all') }} + - test_name: 'webhdfs' + if: ${{ contains(github.event.comment.body, 'webhdfs') || contains(github.event.comment.body, 'all') }} + + update-pr-status: + name: Signal result to PR + runs-on: ubuntu-latest + needs: test-integration + if: always() && (contains(github.event.comment.body, '/ci-run-integration') || contains(github.event.comment.body, '/ci-run-all')) + steps: + - name: Validate issue comment + if: github.event_name == 'issue_comment' + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + - name: (PR comment) Get PR branch + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Submit PR result as ${{ needs.test-integration.result }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ needs.test-integration.result }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4fe09304eba32..f8e26564855e1 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1,34 +1,28 @@ -name: Integration Test Suite +# This workflow is used to run an integration test. +# The most common use case is that it is triggered by another workflow, +# such as the Master Merge Queue Suite, or the Integration Comment. +# +# It can also be triggered on manual dispatch in CI however. +# In that use case, an input for the test name needs to be provided. +# TODO: check if the input is "all" , and run all, without a timeout? + +name: Integration Test on: + workflow_call: + inputs: + if: + required: false + type: boolean + test_name: + required: true + type: string workflow_dispatch: - push: - branches: - - master - paths: - - ".github/workflows/integration-test.yml" - - ".cargo/**" - - "benches/**" - - "lib/**" - - "proto/**" - - "scripts/**" - - "src/**" - - "tests/**" - - "build.rs" - - "Cargo.lock" - - "Cargo.toml" - - "Makefile" - - "rust-toolchain" - pull_request: - types: [opened, synchronize, reopened, labeled] - -concurrency: - # For pull requests, cancel running workflows, for master, run all - # - # `github.event.number` exists for pull requests, otherwise fall back to SHA - # for master - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} - cancel-in-progress: true + inputs: + test_name: + description: "Which integration to test." + required: true + type: string env: AWS_ACCESS_KEY_ID: "dummy" @@ -46,85 +40,36 @@ env: jobs: test-integration: - name: Integration - Linux, ${{ matrix.test }} runs-on: [linux, ubuntu-20.04-8core] - if: | - !github.event.pull_request - || contains(github.event.pull_request.labels.*.name, 'ci-condition: integration tests enable') - strategy: - fail-fast: false - matrix: - include: - - test: 'amqp' - - test: 'appsignal' - - test: 'aws' - - test: 'axiom' - - test: 'azure' - - test: 'clickhouse' - - test: 'databend' - - test: 'datadog-agent' - - test: 'datadog-logs' - - test: 'datadog-metrics' - - test: 'datadog-traces' - - test: 'dnstap' - - test: 'docker-logs' - - test: 'elasticsearch' - - test: 'eventstoredb' - - test: 'fluent' - - test: 'gcp' - - test: 'humio' - - test: 'http-client' - - test: 'influxdb' - - test: 'kafka' - - test: 'logstash' - - test: 'loki' - - test: 'mongodb' - - test: 'nats' - - test: 'nginx' - - test: 'opentelemetry' - - test: 'postgres' - - test: 'prometheus' - - test: 'pulsar' - - test: 'redis' - - test: 'shutdown' - - test: 'splunk' - - test: 'webhdfs' timeout-minutes: 30 + if: inputs.if || github.event_name == 'workflow_dispatch' steps: - - uses: actions/checkout@v3 + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + - run: sudo npm -g install @datadog/datadog-ci - - run: make test-integration-${{ matrix.test }} + + - run: make test-integration-${{ inputs.test_name }} env: TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} - SPLUNK_VERSION: ${{ matrix.env.SPLUNK_VERSION }} + - name: Upload test results run: scripts/upload-test-results.sh if: always() - - run: make test-integration-${{ matrix.test }}-cleanup + + - run: make test-integration-${{ inputs.test_name }}-cleanup if: ${{ always() }} env: TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} - SPLUNK_VERSION: ${{ matrix.env.SPLUNK_VERSION }} - - test-integration-check: - name: test-integration-check - runs-on: ubuntu-20.04 - needs: - - test-integration - steps: - - name: validate - run: echo "OK" - - master-failure: - name: master-failure - if: failure() && github.ref == 'refs/heads/master' - needs: - - test-integration-check - runs-on: ubuntu-20.04 - steps: - - name: Discord notification - env: - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - uses: Ilshidur/action-discord@0.3.2 - with: - args: "Master integration tests failed: " diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000000000..1dc2ef8140160 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,140 @@ +# Integration Test Suite +# +# This workflow runs the integration tests. If the workflow is triggered in the merge queue, all integration tests +# are run. If the workflow is triggered in a PR commit, then the files changed in the PR are evaluated to determine +# if any integration tests will run. + +name: Integration Test Suite + +on: + pull_request: + merge_group: + types: [checks_requested] + +concurrency: + # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue + group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.base_sha }} + cancel-in-progress: true + +env: + AWS_ACCESS_KEY_ID: "dummy" + AWS_SECRET_ACCESS_KEY: "dummy" + CONTAINER_TOOL: "docker" + DD_ENV: "ci" + DD_API_KEY: ${{ secrets.DD_API_KEY }} + RUST_BACKTRACE: full + TEST_LOG: vector=debug + VERBOSE: true + CI: true + PROFILE: debug + # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps + # can be removed when we switch back to the upstream openssl-sys crate + CARGO_NET_GIT_FETCH_WITH_CLI: true + +jobs: + + changes: + if: github.event_name == 'pull_request' + uses: ./.github/workflows/changes.yml + with: + base_ref: ${{ github.event.pull_request.base.ref }} + head_ref: ${{ github.event.pull_request.head.ref }} + secrets: inherit + + # Calls the Integration Test workflow for each integration that was detected to have files changed that impact it. + integration-matrix: + if: always() + uses: ./.github/workflows/integration-test.yml + with: + if: ${{ matrix.run.if }} + test_name: ${{ matrix.run.test_name }} + secrets: inherit + needs: changes + strategy: + fail-fast: false + matrix: + run: + - test_name: 'amqp' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.amqp == 'true' }} + - test_name: 'appsignal' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.appsignal == 'true' }} + - test_name: 'aws' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.aws == 'true' }} + - test_name: 'axiom' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.axiom == 'true' }} + - test_name: 'azure' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.azure == 'true' }} + - test_name: 'clickhouse' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.clickhouse == 'true' }} + - test_name: 'databend' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.databend == 'true' }} + - test_name: 'datadog-agent' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} + - test_name: 'datadog-logs' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} + - test_name: 'datadog-metrics' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} + - test_name: 'datadog-traces' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} + - test_name: 'dnstap' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.dnstap == 'true' }} + - test_name: 'docker-logs' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.docker-logs == 'true' }} + - test_name: 'elasticsearch' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.elasticsearch == 'true' }} + - test_name: 'eventstoredb' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.eventstoredb == 'true' }} + - test_name: 'fluent' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.fluent == 'true' }} + - test_name: 'gcp' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.gcp == 'true' }} + - test_name: 'humio' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.humio == 'true' }} + - test_name: 'http-client' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.http-client == 'true' }} + - test_name: 'influxdb' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.influxdb == 'true' }} + - test_name: 'kafka' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.kafka == 'true' }} + - test_name: 'logstash' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.logstash == 'true' }} + - test_name: 'loki' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.loki == 'true' }} + - test_name: 'mongodb' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.mongodb == 'true' }} + - test_name: 'nats' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.nats == 'true' }} + - test_name: 'nginx' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.nginx == 'true' }} + - test_name: 'opentelemetry' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.opentelemetry == 'true' }} + - test_name: 'postgres' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.postgres == 'true' }} + - test_name: 'prometheus' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.prometheus == 'true' }} + - test_name: 'pulsar' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.pulsar == 'true' }} + - test_name: 'redis' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.redis == 'true' }} + - test_name: 'shutdown' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' }} + - test_name: 'splunk' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.splunk == 'true' }} + - test_name: 'webhdfs' + if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.webhdfs == 'true' }} + + integration: + name: Integration Test Suite + runs-on: ubuntu-latest + needs: + - integration-matrix + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} + steps: + - run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index dd1a0a13ac490..b37e03ba12088 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -1,33 +1,33 @@ +# K8s E2E Suite +# +# This workflow runs under any of the following conditions: +# - manual dispatch in GH UI +# - on a PR commit if the kubernetes_logs source was changed +# - in the merge queue +# - on a schedule at midnight UTC Tue-Sat +# - on demand by either of the following comments in a PR: +# - '/ci-run-k8s' +# - '/ci-run-all' +# +# If the workflow trigger is the nightly schedule, all the k8s versions +# are run in the matrix, otherwise, only the latest is run. + name: K8S E2E Suite on: workflow_dispatch: - push: - branches: - - master - paths: - - ".github/workflows/k8s_e2e.yml" - - ".cargo/**" - - "benches/**" - - "lib/**" - - "proto/**" - - "scripts/**" - - "src/**" - - "tests/**" - - "build.rs" - - "Cargo.lock" - - "Cargo.toml" - - "Makefile" - - "rust-toolchain" - - "distribution/**" pull_request: + issue_comment: + types: [created] + merge_group: + types: [checks_requested] + schedule: + # At midnight UTC Tue-Sat + - cron: '0 0 * * 2-6' concurrency: - # For pull requests, cancel running workflows, for master, run all - # - # `github.event.number` exists for pull requests, otherwise fall back to SHA - # for master - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + group: ${{ github.workflow }}-${{ github.event.number || github.event.issue.id || github.event.merge_group.base_sha || github.event.schedule || github.sha }} + cancel-in-progress: true env: @@ -42,35 +42,82 @@ env: PROFILE: debug jobs: + changes: + if: github.event_name != 'issue_comment' || (github.event.issue.pull_request && + (contains(github.event.comment.body, '/ci-run-k8s') || contains(github.event.comment.body, '/ci-run-all'))) + uses: ./.github/workflows/changes.yml + with: + base_ref: ${{ github.event.pull_request.base.ref }} + head_ref: ${{ github.event.pull_request.head.ref }} + secrets: inherit + build-x86_64-unknown-linux-gnu: name: Build - x86_64-unknown-linux-gnu runs-on: [linux, ubuntu-20.04-8core] - if: | - !github.event.pull_request - || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e tests enable') - || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e all targets') + needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true' # cargo-deb requires a release build, but we don't need optimizations for tests env: CARGO_PROFILE_RELEASE_OPT_LEVEL: 0 CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 256 CARGO_INCREMENTAL: 0 steps: - - uses: actions/checkout@v3 + - name: Validate issue comment + if: github.event_name == 'issue_comment' + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + - uses: actions/cache@v3 with: path: | ~/.cargo/registry ~/.cargo/git key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - run: bash scripts/environment/prepare.sh - run: echo "::add-matcher::.github/matchers/rust.json" - run: VECTOR_VERSION="$(cargo vdev version)" make package-deb-x86_64-unknown-linux-gnu + - uses: actions/upload-artifact@v3 with: name: e2e-test-deb-package path: target/artifacts/* + - name: (PR comment) Set latest commit status as 'failure' + uses: myrotvorets/set-commit-status-action@1.1.6 + if: failure() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: 'failure' + # GitHub Actions don't support `matrix` at the job-level `if:` condition. # We apply this workaround - compute `matrix` in a preceding job, and assign # it's value dynamically at the actual test job. @@ -80,13 +127,19 @@ jobs: compute-k8s-test-plan: name: Compute K8s test plan runs-on: [linux, ubuntu-20.04-8core] + needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true' outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} - if: | - !github.event.pull_request - || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e tests enable') - || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e all targets') steps: + - name: Validate issue comment + if: github.event_name == 'issue_comment' + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + - uses: actions/github-script@v6.4.1 id: set-matrix with: @@ -101,11 +154,11 @@ jobs: // https://cloud.google.com/kubernetes-engine/docs/release-notes // https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#aks-kubernetes-release-calendar const kubernetes_version = [ - { version: "v1.23.3", is_essential: true }, - { version: "v1.22.5", is_essential: true }, - { version: "v1.21.8", is_essential: true }, - { version: "v1.20.14", is_essential: true }, - { version: "v1.19.8" }, + { version: "v1.23.3", is_essential: true }, + { version: "v1.22.5", is_essential: false }, + { version: "v1.21.8", is_essential: false }, + { version: "v1.20.14", is_essential: false }, + { version: "v1.19.8", is_essential: false }, ] const container_runtime = [ "docker", @@ -113,15 +166,10 @@ jobs: // https://github.com/kubernetes/minikube/issues/12928 // "crio", ] - const ci_condition_label = 'ci-condition: k8s e2e all targets' - // Planing. - const is_in_pull_request = !!context.payload.pull_request; - const should_test_all_targets = ( - !is_in_pull_request || - context.payload.pull_request.labels.some(label => label.name === ci_condition_label) - ) - const filter_targets = array => array.filter(val => should_test_all_targets || val.is_essential) + // Run all versions if triggered by nightly schedule. Otherwise only run latest. + const run_all = context.eventName == "schedule"; + const filter_targets = array => array.filter(val => run_all || val.is_essential) const matrix = { minikube_version, @@ -147,36 +195,82 @@ jobs: matrix: ${{ fromJson(needs.compute-k8s-test-plan.outputs.matrix) }} fail-fast: false steps: - - name: Checkout + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 with: name: e2e-test-deb-package path: target/artifacts + - name: Setup Minikube run: scripts/ci-setup-minikube.sh env: KUBERNETES_VERSION: ${{ matrix.kubernetes_version.version }} MINIKUBE_VERSION: ${{ matrix.minikube_version }} CONTAINER_RUNTIME: ${{ matrix.container_runtime }} + - run: make test-e2e-kubernetes env: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" CARGO_INCREMENTAL: 0 - master-failure: - name: master-failure - if: failure() && github.ref == 'refs/heads/master' - needs: - - build-x86_64-unknown-linux-gnu - - compute-k8s-test-plan - - test-e2e-kubernetes - runs-on: ubuntu-20.04 + - name: (PR comment) Set latest commit status as failure + uses: myrotvorets/set-commit-status-action@1.1.6 + if: failure() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: 'failure' + + final-result: + name: K8s E2E Suite + runs-on: ubuntu-latest + needs: test-e2e-kubernetes + if: | + always() && (github.event_name != 'issue_comment' || (github.event.issue.pull_request + && (contains(github.event.comment.body, '/ci-run-k8s') || contains(github.event.comment.body, '/ci-run-all')))) + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - - name: Discord notification - env: - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - uses: Ilshidur/action-discord@0.3.2 - with: - args: "Master k8s e2e tests failed: " + - name: Validate issue comment + if: github.event_name == 'issue_comment' + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + - name: (PR comment) Get PR branch + if: success() && github.event_name == 'issue_comment' + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Submit PR result as success + if: success() && github.event_name == 'issue_comment' + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + status: 'success' + + - run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/master_merge_queue.yml b/.github/workflows/master_merge_queue.yml new file mode 100644 index 0000000000000..6e38bd7f29a24 --- /dev/null +++ b/.github/workflows/master_merge_queue.yml @@ -0,0 +1,153 @@ +# Master Merge Queue Test Suite +# +# This workflow orchestrates a collection of workflows that are required for the merge queue check. +# +# Most of the workflows that are jobs within this one, are able to be run on demand +# by issuing a PR comment with the respective command to trigger said workflow. +# +# The design of this workflow relies on the first real job "changes" to detect file +# changes against the base, and each downstream workflow after that will only be +# called if the files for that area have changed. +# + +name: Master Merge Queue Test Suite + +on: + # Only want to run this on merge queue, but because GH doesn't allow specifying different required checks + # for pull request and merge queue, we need to "run" it in pull request, but in the jobs we will just auto pass. + pull_request: + merge_group: + types: [checks_requested] + +concurrency: + # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue + group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.base_sha }} + cancel-in-progress: true + +env: + AWS_ACCESS_KEY_ID: "dummy" + AWS_SECRET_ACCESS_KEY: "dummy" + CONTAINER_TOOL: "docker" + DD_ENV: "ci" + DD_API_KEY: ${{ secrets.DD_API_KEY }} + RUST_BACKTRACE: full + TEST_LOG: vector=debug + VERBOSE: true + CI: true + PROFILE: debug + # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps + # can be removed when we switch back to the upstream openssl-sys crate + CARGO_NET_GIT_FETCH_WITH_CLI: true + +jobs: + # This is the entry job which is required for all the actual tests in this workflow. + # If we don't run this job (such as in a pull request), then by consequence all downstream + # test jobs are not run. This allows us to not have to check for merge group in each job. + changes: + if: ${{ github.event_name == 'merge_group' }} + uses: ./.github/workflows/changes.yml + with: + base_ref: ${{ github.event.merge_group.base_ref }} + head_ref: ${{ github.event.merge_group.head_ref }} + secrets: inherit + + test-cli: + if: needs.changes.outputs.source == 'true' + uses: ./.github/workflows/cli.yml + needs: changes + secrets: inherit + + test-misc: + if: needs.changes.outputs.source == 'true' + uses: ./.github/workflows/misc.yml + needs: changes + secrets: inherit + + test-environment: + uses: ./.github/workflows/environment.yml + needs: changes + secrets: inherit + + check-msrv: + if: needs.changes.outputs.source == 'true' + uses: ./.github/workflows/msrv.yml + needs: changes + secrets: inherit + + check-component-features: + if: needs.changes.outputs.source == 'true' + uses: ./.github/workflows/component_features.yml + needs: changes + secrets: inherit + + cross-linux: + # We run cross checks when dependencies change to ensure they still build. + # This helps us avoid adopting dependencies that aren't compatible with other architectures. + if: needs.changes.outputs.dependencies == 'true' + uses: ./.github/workflows/cross.yml + needs: changes + secrets: inherit + + unit-mac: + if: needs.changes.outputs.source == 'true' + uses: ./.github/workflows/unit_mac.yml + needs: changes + secrets: inherit + + unit-windows: + if: needs.changes.outputs.source == 'true' + uses: ./.github/workflows/unit_windows.yml + needs: changes + secrets: inherit + + install-sh: + if: needs.changes.outputs.install == 'true' + uses: ./.github/workflows/install-sh.yml + needs: changes + secrets: inherit + + # TODO: in a followup PR, run the regression workflow here, as a single reusable workflow. + # + # NOTE: This design of passing in the pr-number to the Regression workflow requires that the merge queue + # be configured contain a maximum of one PR per execution. This is so that the regression report generated + # by the workflow can be posted as a comment to the PR. + # At a later time, we may want to revisit this in order to allow multiple PRs to be included in a merge + # queue execution. At such time, the logic of uploading of the report will need to change to account for + # multiple PRs. + # regression: + # if: needs.changes.outputs.source == 'true' + # uses: ./.github/workflows/regression.yml + # with: + # pr_number: ${{ needs.changes.outputs.pr-number }} + # base_sha: ${{ github.event.merge_group.base_sha }} + # head_sha: ${{ github.event.merge_group.head_sha }} + # needs: changes + # secrets: inherit + + master-merge-queue-check: + name: Master Merge Queue Suite + # Always run this so that pull_request triggers are marked as success. + if: always() + runs-on: ubuntu-20.04 + needs: + - changes + - test-cli + - test-misc + - test-environment + - check-msrv + - check-component-features + - cross-linux + - unit-mac + - unit-windows + - install-sh + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} + steps: + - name: exit + run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml new file mode 100644 index 0000000000000..f66c577ec7c54 --- /dev/null +++ b/.github/workflows/misc.yml @@ -0,0 +1,62 @@ +name: Miscellaneous - Linux + +on: + workflow_call: + +jobs: + test-misc: + runs-on: [linux, ubuntu-20.04-8core] + env: + CARGO_INCREMENTAL: 0 + steps: + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Miscellaneous - Linux + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + name: Cache Cargo registry + index + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: bash scripts/environment/prepare.sh + - run: echo "::add-matcher::.github/matchers/rust.json" + - run: make test-behavior + - run: make check-examples + - run: make test-docs + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@1.1.6 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Miscellaneous - Linux + status: ${{ job.status }} diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml new file mode 100644 index 0000000000000..361aa9cd4a3b8 --- /dev/null +++ b/.github/workflows/msrv.yml @@ -0,0 +1,21 @@ +name: Check minimum supported Rust version + +on: + workflow_call: + +env: + RUST_BACKTRACE: full + CI: true + PROFILE: debug + # observing issues fetching boringssl via HTTPS in the OSX build, seeing if this helps + # can be removed when we switch back to the upstream openssl-sys crate + CARGO_NET_GIT_FETCH_WITH_CLI: true + +jobs: + check-msrv: + runs-on: [ubuntu-20.04] + steps: + - uses: actions/checkout@v3 + - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh + - run: cargo install cargo-msrv --version 0.15.1 + - run: cargo msrv verify diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index ec11abd37f65c..40764c5586577 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -1,86 +1,246 @@ -# Regression Detection +# Regression Detection Suite +# +# This workflow runs under the following conditions: +# - in the merge queue if any source files were modified, added or deleted. +# - on demand by a PR comment matching either of: +# - '/ci-run-regression' +# - '/ci-run-all' +# (the comment issuer must be a member of the Vector GH team) # # This workflow runs our regression detection experiments, which are relative -# evaluations of the base SHA for the PR to whatever SHA was just pushed into -# the project (unless that SHA happens to be master branch HEAD). The goal is to -# give quick-ish feedback on all-up Vector for a variety of configs as to -# whether throughput performance has gone down, gotten more variable in the +# evaluations of the base SHA and head SHA, whose determination depends on how +# the workflow is invoked. +# +# The goal is to give quick-ish feedback on all-up Vector for a variety of configs +# as to whether throughput performance has gone down, gotten more variable in the # pushed SHA. # # Regression detection is always done relative to the pushed SHA, meaning any # changes you introduce to the experiment will be picked up both for the base -# SHA variant and your current SHA. Tags are SHA-SHA. The first SHA is the one -# that triggered this workflow, the second is the one of the Vector being -# tested. For comparison images the two SHAs are identical. +# SHA variant and your current SHA. +# +# Docker image tags are SHA-SHA. The first SHA is the one that triggered this +# workflow, the second is the one of the Vector being tested. +# For comparison images the two SHAs are identical. -name: Regression Detector +name: Regression Detection Suite on: - pull_request: - paths-ignore: - - "docs/**" - - "rfcs/**" - - "website/**" merge_group: types: [checks_requested] + workflow_call: + # Don't want to run this on each PR commit, but because GH doesn't allow specifying different required checks + # for pull request and merge queue, we need to "run" it in pull request, but in the jobs we will just auto pass. + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.id || github.event.merge_group.base_sha || github.sha }} + cancel-in-progress: true jobs: - cancel-previous: - runs-on: ubuntu-22.04 - timeout-minutes: 3 + + # Only run this workflow if files changed in areas that could possibly introduce a regression + should-run: + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + outputs: + source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} + comment_valid: ${{ steps.comment.outputs.isTeamMember }} steps: - - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ secrets.GITHUB_TOKEN }} - all_but_latest: true # can cancel workflows scheduled later + - uses: actions/checkout@v3 + + - name: Collect file changes + id: changes + if: github.event_name == 'merge_group' + uses: dorny/paths-filter@v2 + with: + base: ${{ github.event.merge_group.base_ref }} + ref: ${{ github.event.merge_group.head_ref }} + list-files: shell + filters: | + all_changed: + - added|deleted|modified: "**" + ignore: + - "distribution/**" + - "rust-doc/**" + - "docs/**" + - "rfcs/**" + - "testing/**" + - "tilt/**" + - "website/**" + - "*.md" + - "Tiltfile" + - "netlify.toml" + - "NOTICE" + - "LICENSE-3rdparty.csv" + - "LICENSE" + + # This step allows us to conservatively run the tests if we added a new + # file or directory for source code, but forgot to add it to this workflow. + # Instead, we may unnecessarily run the test on new file or dir additions that + # wouldn't likely introduce regressions. + - name: Determine if should not run due to irrelevant file changes + id: filter + if: github.event_name == 'merge_group' + env: + ALL: ${{ steps.changes.outputs.all_changed_files }} + IGNORE: ${{ steps.changes.outputs.ignore_files }} + run: | + echo "ALL='${{ env.ALL }}'" + echo "IGNORE='${{ env.IGNORE }}'" + export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}")) + echo "SOURCE_CHANGED='${SOURCE_CHANGED}'" + + if [ "${SOURCE_CHANGED}" == "" ]; then + export SOURCE_CHANGED="false" + else + export SOURCE_CHANGED="true" + fi + + echo "SOURCE_CHANGED='${SOURCE_CHANGED}'" + echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT compute-metadata: - name: Compute metadata for regression experiments + name: Compute metadata runs-on: ubuntu-22.04 + needs: should-run + if: github.event_name != 'merge_group' || needs.should-run.outputs.source_changed == 'true' outputs: - pr-number: ${{ steps.pr-metadata.outputs.PR_NUMBER }} + pr-number: ${{ steps.pr-metadata-merge-queue.outputs.PR_NUMBER || steps.pr-metadata-comment.outputs.PR_NUMBER }} + baseline-sha: ${{ steps.pr-metadata-merge-queue.outputs.BASELINE_SHA || steps.pr-metadata-comment.outputs.BASELINE_SHA }} + baseline-tag: ${{ steps.pr-metadata-merge-queue.outputs.BASELINE_TAG || steps.pr-metadata-comment.outputs.BASELINE_TAG }} + comparison-sha: ${{ steps.pr-metadata-merge-queue.outputs.COMPARISON_SHA || steps.pr-metadata-comment.outputs.COMPARISON_SHA }} + comparison-tag: ${{ steps.pr-metadata-merge-queue.outputs.COMPARISON_TAG || steps.pr-metadata-comment.outputs.COMPARISON_TAG }} + + # below are used in the experiment/analyze jobs + cpus: ${{ steps.system.outputs.CPUS }} + memory: ${{ steps.system.outputs.MEMORY }} + vector-cpus: ${{ steps.system.outputs.VECTOR_CPUS }} - comparison-sha: ${{ steps.comparison.outputs.COMPARISON }} - comparison-tag: ${{ steps.comparison.outputs.COMPARISON_TAG }} - baseline-sha: ${{ steps.baseline.outputs.BASELINE }} - baseline-tag: ${{ steps.baseline.outputs.BASELINE_TAG }} + replicas: ${{ steps.experimental-meta.outputs.REPLICAS }} + warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }} + total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }} + p-value: ${{ steps.experimental-meta.outputs.P_VALUE }} + smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} + lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }} steps: + - uses: actions/checkout@v3 - with: - ref: ${{ github.base_ref }} - path: baseline-vector - - name: Setup PR metadata - id: pr-metadata + # If triggered by issue comment, the event payload doesn't directly contain the head and base sha from the PR. + # But, we can retrieve this info from some commands. + - name: Get PR metadata (issue_comment) + id: pr-metadata-comment + if: github.event_name == 'issue_comment' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_OUTPUT + export PR_NUMBER=${{ github.event.issue.number }} + echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT - - name: Setup baseline variables - id: baseline - run: | - pushd baseline-vector - export BASELINE_SHA=$(git rev-parse HEAD) - popd + gh pr checkout ${PR_NUMBER} + + export BASELINE_SHA=$(git merge-base --fork-point master) + echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT + + export COMPARISON_SHA=$(git rev-parse HEAD) + echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT + + export BASELINE_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${BASELINE_SHA}" + echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT + + export COMPARISON_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${COMPARISON_SHA}" + echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT + + echo "pr number is: ${PR_NUMBER}" - export BASELINE_TAG="${{ github.event.pull_request.head.sha }}-${BASELINE_SHA}" echo "baseline sha is: ${BASELINE_SHA}" echo "baseline tag is: ${BASELINE_TAG}" - echo "BASELINE=${BASELINE_SHA}" >> $GITHUB_OUTPUT - echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT + echo "comparison sha is: ${COMPARISON_SHA}" + echo "comparison tag is: ${COMPARISON_TAG}" - - name: Setup comparison variables - id: comparison + # If triggered by merge queue, the PR number is not available in the payload. While we restrict the number of PRs in the + # queue to 1, we can get the PR number by parsing the merge queue temp branch's ref. + - name: Get PR metadata (merge queue) + id: pr-metadata-merge-queue + if: github.event_name != 'issue_comment' run: | - export COMPARISON_SHA=${{ github.event.pull_request.head.sha }} - export COMPARISON_TAG="${{ github.event.pull_request.head.sha }}-${{ github.event.pull_request.head.sha }}" + export PR_NUMBER=$(echo "${{ github.ref }}" | sed -n 's|^refs/heads/gh-readonly-queue/master/pr-\([0-9]*\)-.*$|\1|p') + echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT + + export BASELINE_SHA=${{ github.event.merge_group.base_sha }} + echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT + + export COMPARISON_SHA=${{ github.event.merge_group.head_sha }} + echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT + + export BASELINE_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${BASELINE_SHA}" + echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT + + export COMPARISON_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${COMPARISON_SHA}" + echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT + + echo "pr number is: ${PR_NUMBER}" + + echo "baseline sha is: ${BASELINE_SHA}" + echo "baseline tag is: ${BASELINE_TAG}" echo "comparison sha is: ${COMPARISON_SHA}" echo "comparison tag is: ${COMPARISON_TAG}" - echo "COMPARISON=${COMPARISON_SHA}" >> $GITHUB_OUTPUT - echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT + - name: Setup experimental metadata + id: experimental-meta + run: | + export WARMUP_SECONDS="45" + export REPLICAS="10" + export TOTAL_SAMPLES="600" + export P_VALUE="0.1" + export SMP_CRATE_VERSION="0.7.3" + export LADING_VERSION="0.12.0" + + echo "warmup seconds: ${WARMUP_SECONDS}" + echo "replicas: ${REPLICAS}" + echo "total samples: ${TOTAL_SAMPLES}" + echo "regression p-value: ${P_VALUE}" + echo "smp crate version: ${SMP_CRATE_VERSION}" + echo "lading version: ${LADING_VERSION}" + + echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT + echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT + echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT + echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT + echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT + echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT + + - name: Setup system details + id: system + run: | + export CPUS="7" + export MEMORY="30g" + export VECTOR_CPUS="4" + + echo "cpus total: ${CPUS}" + echo "memory total: ${MEMORY}" + echo "vector cpus: ${VECTOR_CPUS}" + echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT + echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT + echo "VECTOR_CPUS=${VECTOR_CPUS}" >> $GITHUB_OUTPUT + + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.pr-metadata-comment.outputs.COMPARISON_SHA }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Regression Detection Suite + status: pending ## ## BUILD ## @@ -114,7 +274,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar tags: | - vector:${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.baseline-tag }} + vector:${{ needs.compute-metadata.outputs.baseline-tag }} - name: Upload image as artifact uses: actions/upload-artifact@v3 @@ -151,7 +311,7 @@ jobs: builder: ${{ steps.buildx.outputs.name }} outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar tags: | - vector:${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.comparison-tag }} + vector:${{ needs.compute-metadata.outputs.comparison-tag }} - name: Upload image as artifact uses: actions/upload-artifact@v3 @@ -159,25 +319,472 @@ jobs: name: comparison-image path: "${{ runner.temp }}/comparison-image.tar" - transmit-metadata: - name: Transmit metadata to trusted workflow + confirm-valid-credentials: + name: Confirm AWS credentials are minimally valid + runs-on: ubuntu-22.04 + needs: + - compute-metadata + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + ## + ## SUBMIT + ## + + upload-baseline-image-to-ecr: + name: Upload baseline images to ECR + runs-on: ubuntu-22.04 + needs: + - compute-metadata + - confirm-valid-credentials + - build-baseline + steps: + - name: 'Download baseline image' + uses: actions/download-artifact@v3 + with: + name: baseline-image + + - name: Load baseline image + run: | + docker load --input baseline-image.tar + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Docker Login to ECR + uses: docker/login-action@v2 + with: + registry: ${{ steps.login-ecr.outputs.registry }} + + - name: Tag & push baseline image + run: | + docker tag vector:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} + docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} + + upload-comparison-image-to-ecr: + name: Upload comparison images to ECR + runs-on: ubuntu-22.04 + needs: + - compute-metadata + - confirm-valid-credentials + - build-comparison + steps: + - name: 'Download comparison image' + uses: actions/download-artifact@v3 + with: + name: comparison-image + + - name: Load comparison image + run: | + docker load --input comparison-image.tar + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Docker Login to ECR + uses: docker/login-action@v2 + with: + registry: ${{ steps.login-ecr.outputs.registry }} + + - name: Tag & push comparison image + run: | + docker tag vector:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} + docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} + + submit-job: + name: Submit regression job + runs-on: ubuntu-22.04 + needs: + - compute-metadata + - upload-baseline-image-to-ecr + - upload-comparison-image-to-ecr + steps: + - name: Check status, in-progress + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='pending' \ + -f description='Experiments submitted to the Regression Detection cluster.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - uses: actions/checkout@v3 + with: + ref: ${{ needs.compute-metadata.outputs.comparison-sha }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + - name: Submit job + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ + --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ + --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ + --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ + --replicas ${{ needs.compute-metadata.outputs.replicas }} \ + --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} \ + --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} \ + --baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \ + --comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \ + --target-command "/usr/local/bin/vector" \ + --target-config-dir ${{ github.workspace }}/regression/ \ + --target-cpu-allotment "${{ needs.compute-metadata.outputs.cpus }}" \ + --target-memory-allotment "${{ needs.compute-metadata.outputs.memory }}" \ + --target-environment-variables "VECTOR_THREADS=${{ needs.compute-metadata.outputs.vector-cpus }},VECTOR_REQUIRE_HEALTHY=true" \ + --target-name vector \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - uses: actions/upload-artifact@v3 + with: + name: vector-submission-metadata + path: ${{ runner.temp }}/submission-metadata + + - name: Await job + timeout-minutes: 120 + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ + --wait \ + --wait-delay-seconds 60 \ + --wait-timeout-minutes 90 \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - name: Handle cancellation if necessary + if: ${{ cancelled() }} + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - name: Check status, cancelled + if: ${{ cancelled() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Experiments submitted to the Regression Detection cluster cancelled.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, success + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Experiments submitted to the Regression Detection cluster successfully.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Experiments submitted to the Regression Detection Suite failed.' \ + -f context='Regression Detection Suite / submission' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + ## + ## ANALYZE + ## + + detect-regression: + name: Determine regression status runs-on: ubuntu-22.04 needs: + - submit-job - compute-metadata steps: - - name: Write out metadata - run: | - echo "COMPARISON_TAG=${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.comparison-tag }}" > ${{ runner.temp }}/meta - echo "COMPARISON_SHA=${{ needs.compute-metadata.outputs.comparison-sha }}" >> ${{ runner.temp }}/meta - echo "BASELINE_TAG=${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.baseline-tag }}" >> ${{ runner.temp }}/meta - echo "BASELINE_SHA=${{ needs.compute-metadata.outputs.baseline-sha }}" >> ${{ runner.temp }}/meta - echo "CHECKOUT_SHA=${{ github.sha }}" >> ${{ runner.temp }}/meta - echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> ${{ runner.temp }}/meta - echo "BASE_SHA=${{ github.event.pull_request.base.sha }}" >> ${{ runner.temp }}/meta - echo "GITHUB_EVENT_NUMBER=${{ github.event.number }}" >> ${{ runner.temp }}/meta - - - name: Upload metadata + - uses: actions/checkout@v3 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + - name: Download submission metadata + uses: actions/download-artifact@v3 + with: + name: vector-submission-metadata + path: ${{ runner.temp }}/ + + - name: Determine if PR introduced a regression + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \ + --submission-metadata ${{ runner.temp }}/submission-metadata + + - name: Check status, cancelled + if: ${{ cancelled() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite cancelled.' \ + -f context='Regression Detection Suite / detect-regression' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, success + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Analyze experimental results from Regression Detection Suite succeeded.' \ + -f context='Regression Detection Suite / detect-regression' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite failed.' \ + -f context='Regression Detection Suite / detect-regression' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + analyze-experiment: + name: Download regression analysis & upload report + runs-on: ubuntu-22.04 + needs: + - submit-job + - compute-metadata + steps: + - name: Check status, in-progress + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='pending' \ + -f description='Analyze experimental results from Regression Detection Suite.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - uses: actions/checkout@v3 + with: + ref: ${{ needs.compute-metadata.outputs.comparison-sha }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2.0.0 + with: + aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Download SMP binary + run: | + aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp + + - name: Download submission metadata + uses: actions/download-artifact@v3 + with: + name: vector-submission-metadata + path: ${{ runner.temp }}/ + + - name: Sync regression report to local system + env: + RUST_LOG: info + run: | + chmod +x ${{ runner.temp }}/bin/smp + + ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ + --submission-metadata ${{ runner.temp }}/submission-metadata \ + --output-path "${{ runner.temp }}/outputs" + + - name: Read regression report + id: read-analysis + uses: juliangruber/read-file-action@v1 + with: + path: ${{ runner.temp }}/outputs/report.html + + - name: Post report to PR + uses: peter-evans/create-or-update-comment@v3 + with: + issue-number: ${{ needs.compute-metadata.outputs.pr-number }} + edit-mode: append + body: ${{ steps.read-analysis.outputs.content }} + + - name: Upload regression report to artifacts uses: actions/upload-artifact@v3 with: - name: meta - path: "${{ runner.temp }}/meta" + name: capture-artifacts + path: ${{ runner.temp }}/outputs/* + + - name: Check status, cancelled + if: ${{ cancelled() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite cancelled.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, success + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='success' \ + -f description='Analyze experimental results from Regression Detection Suite succeeded.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Check status, failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \ + -f state='failure' \ + -f description='Analyze experimental results from Regression Detection Suite failed.' \ + -f context='Regression Detection Suite / analyze-experiment' \ + -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + # This job always runs- if an issue_comment triggered it, we need to update the check status in the PR, + # and if a pull_request triggered it, we need to flag the check status as a success. + regression-detection-suite: + name: Regression Detection Suite + runs-on: ubuntu-latest + if: always() + needs: + - compute-metadata + - build-baseline + - build-comparison + - confirm-valid-credentials + - upload-baseline-image-to-ecr + - upload-comparison-image-to-ecr + - submit-job + - detect-regression + - analyze-experiment + env: + FAILED: ${{ contains(needs.*.result, 'failure') }} + steps: + - name: (PR comment) Get PR branch + if: github.event_name == 'issue_comment' + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Submit PR result as failed + if: github.event_name == 'issue_comment' && env.FAILED == 'true' + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.compute-metadata.outputs.comparison-sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Regression Detection Suite + status: 'failure' + + - name: (PR comment) Submit PR result as success + if: github.event_name == 'issue_comment' && env.FAILED != 'true' + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Regression Detection Suite + status: 'success' + + - name: exit + run: | + echo "failed=${{ env.FAILED }}" + if [[ "$FAILED" == "true" ]] ; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/regression_trusted.yml b/.github/workflows/regression_trusted.yml deleted file mode 100644 index 52de4d14d3236..0000000000000 --- a/.github/workflows/regression_trusted.yml +++ /dev/null @@ -1,595 +0,0 @@ -name: Regression Detector (trusted) - -on: - workflow_run: - workflows: ["Regression Detector"] - types: - - completed - -jobs: - compute-metadata: - name: Compute metadata for regression experiments - runs-on: ubuntu-22.04 - if: > - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' - outputs: - cpus: ${{ steps.system.outputs.CPUS }} - memory: ${{ steps.system.outputs.MEMORY }} - vector-cpus: ${{ steps.system.outputs.VECTOR_CPUS }} - - comparison-sha: ${{ steps.metadata.outputs.COMPARISON_SHA }} - comparison-tag: ${{ steps.metadata.outputs.COMPARISON_TAG }} - baseline-sha: ${{ steps.metadata.outputs.BASELINE_SHA }} - baseline-tag: ${{ steps.metadata.outputs.BASELINE_TAG }} - head-sha: ${{ steps.metadata.outputs.HEAD_SHA }} - checkout-sha: ${{ steps.metadata.outputs.CHECKOUT_SHA }} - github-event-number: ${{ steps.metadata.outputs.GITHUB_EVENT_NUMBER }} - - replicas: ${{ steps.experimental-meta.outputs.REPLICAS }} - warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }} - total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }} - p-value: ${{ steps.experimental-meta.outputs.P_VALUE }} - smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} - lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }} - - steps: - - name: Setup experimental metadata - id: experimental-meta - run: | - export WARMUP_SECONDS="45" - export REPLICAS="10" - export TOTAL_SAMPLES="600" - export P_VALUE="0.1" - export SMP_CRATE_VERSION="0.7.3" - export LADING_VERSION="0.12.0" - - echo "warmup seconds: ${WARMUP_SECONDS}" - echo "replicas: ${REPLICAS}" - echo "total samples: ${TOTAL_SAMPLES}" - echo "regression p-value: ${P_VALUE}" - echo "smp crate version: ${SMP_CRATE_VERSION}" - echo "lading version: ${LADING_VERSION}" - - echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT - echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT - echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT - echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT - echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT - echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT - - - name: Setup system details - id: system - run: | - export CPUS="7" - export MEMORY="30g" - export VECTOR_CPUS="4" - - echo "cpus total: ${CPUS}" - echo "memory total: ${MEMORY}" - echo "vector cpus: ${VECTOR_CPUS}" - - echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT - echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT - echo "VECTOR_CPUS=${VECTOR_CPUS}" >> $GITHUB_OUTPUT - - # github.rest.actions.listWorkflowRunArtifacts only returns first 30 - # artifacts, and returns a { data, headers, status, url } object. The - # "data" part of this object contains the artifact data we care about. - # The fields of this data object correspond to the fields in the - # "Example Response" JSON object in - # https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts. - # To return more than 30 responses, use the github.paginate API in - # https://octokit.github.io/rest.js/v19#custom-requests - # `github-script` aliases `octokit` to the `github` namespace. - - name: 'Download metadata' - uses: actions/github-script@v6.4.1 - with: - script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "meta" - })[0]; - - console.log("Downloading artifact %s", matchArtifact.id); - - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/meta.zip', Buffer.from(download.data)); - - - run: unzip meta.zip - - - name: Setup metadata - id: metadata - run: | - cat meta - cat meta >> $GITHUB_OUTPUT - - confirm-valid-credentials: - name: Confirm AWS credentials are minimally valid - runs-on: ubuntu-22.04 - needs: - - compute-metadata - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2.0.0 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - ## - ## SUBMIT - ## - - upload-baseline-image-to-ecr: - name: Upload images to ECR - runs-on: ubuntu-22.04 - needs: - - compute-metadata - - confirm-valid-credentials - steps: - # github.rest.actions.listWorkflowRunArtifacts only returns first 30 - # artifacts, and returns a { data, headers, status, url } object. The - # "data" part of this object contains the artifact data we care about. - # The fields of this data object correspond to the fields in the - # "Example Response" JSON object in - # https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts. - # To return more than 30 responses, use the github.paginate API in - # https://octokit.github.io/rest.js/v19#custom-requests - # `github-script` aliases `octokit` to the `github` namespace. - - name: 'Download baseline image' - uses: actions/github-script@v6.4.1 - with: - script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "baseline-image" - })[0]; - - console.log("Downloading artifact %s", matchArtifact.id); - - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/baseline-image.zip', Buffer.from(download.data)); - - - run: unzip baseline-image.zip - - - name: Load baseline image - run: | - docker load --input baseline-image.tar - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2.0.0 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Docker Login to ECR - uses: docker/login-action@v2 - with: - registry: ${{ steps.login-ecr.outputs.registry }} - - - name: Tag & push baseline image - run: | - docker tag vector:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} - docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} - - upload-comparison-image-to-ecr: - name: Upload images to ECR - runs-on: ubuntu-22.04 - needs: - - compute-metadata - - confirm-valid-credentials - steps: - # github.rest.actions.listWorkflowRunArtifacts only returns first 30 - # artifacts, and returns a { data, headers, status, url } object. The - # "data" part of this object contains the artifact data we care about. - # The fields of this data object correspond to the fields in the - # "Example Response" JSON object in - # https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts. - # To return more than 30 responses, use the github.paginate API in - # https://octokit.github.io/rest.js/v19#custom-requests - # `github-script` aliases `octokit` to the `github` namespace. - - name: 'Download comparison image' - uses: actions/github-script@v6.4.1 - with: - script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "comparison-image" - })[0]; - - console.log("Downloading artifact %s", matchArtifact.id); - - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/comparison-image.zip', Buffer.from(download.data)); - - - run: unzip comparison-image.zip - - - name: Load comparison image - run: | - docker load --input comparison-image.tar - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2.0.0 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Docker Login to ECR - uses: docker/login-action@v2 - with: - registry: ${{ steps.login-ecr.outputs.registry }} - - - name: Tag & push comparison image - run: | - docker tag vector:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} - docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} - - submit-job: - name: Submit regression job - runs-on: ubuntu-22.04 - needs: - - compute-metadata - - upload-baseline-image-to-ecr - - upload-comparison-image-to-ecr - steps: - - name: Check status, in-progress - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='pending' \ - -f description='Experiments submitted to the Regression Detector cluster.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.checkout-sha }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2.0.0 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - - name: Submit job - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ - --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ - --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ - --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ - --replicas ${{ needs.compute-metadata.outputs.replicas }} \ - --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} \ - --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} \ - --baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \ - --comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \ - --target-command "/usr/local/bin/vector" \ - --target-config-dir ${{ github.workspace }}/regression/ \ - --target-cpu-allotment "${{ needs.compute-metadata.outputs.cpus }}" \ - --target-memory-allotment "${{ needs.compute-metadata.outputs.memory }}" \ - --target-environment-variables "VECTOR_THREADS=${{ needs.compute-metadata.outputs.vector-cpus }},VECTOR_REQUIRE_HEALTHY=true" \ - --target-name vector \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - uses: actions/upload-artifact@v3 - with: - name: vector-submission-metadata - path: ${{ runner.temp }}/submission-metadata - - - name: Await job - timeout-minutes: 120 - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ - --wait \ - --wait-delay-seconds 60 \ - --wait-timeout-minutes 90 \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Handle cancellation if necessary - if: ${{ cancelled() }} - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Check status, cancelled - if: ${{ cancelled() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='failure' \ - -f description='Experiments submitted to the Regression Detector cluster cancelled.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, success - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Experiments submitted to the Regression Detector cluster successfully.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, failure - if: ${{ failure() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Experiments submitted to the Regression Detector cluster failed.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - ## - ## ANALYZE - ## - - detect-regression: - name: Determine regression status - runs-on: ubuntu-22.04 - needs: - - submit-job - - compute-metadata - steps: - - uses: actions/checkout@v3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2.0.0 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - - name: Download submission metadata - uses: actions/download-artifact@v3 - with: - name: vector-submission-metadata - path: ${{ runner.temp }}/ - - - name: Determine if PR introduced a regression - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Check status, cancelled - if: ${{ cancelled() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='failure' \ - -f description='Analyze experimental results from Regression Detector cancelled.' \ - -f context='Regression Detector / detect-regression' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, success - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Analyze experimental results from Regression Detector succeeded.' \ - -f context='Regression Detector / detect-regression' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, failure - if: ${{ failure() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Analyze experimental results from Regression Detector failed.' \ - -f context='Regression Detector / detect-regression' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - analyze-experiment: - name: Download regression analysis & upload report - runs-on: ubuntu-22.04 - needs: - - submit-job - - compute-metadata - steps: - - name: Check status, in-progress - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='pending' \ - -f description='Analyze experimental results from Regression Detector.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.checkout-sha }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2.0.0 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - - name: Download submission metadata - uses: actions/download-artifact@v3 - with: - name: vector-submission-metadata - path: ${{ runner.temp }}/ - - - name: Sync regression report to local system - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ - --submission-metadata ${{ runner.temp }}/submission-metadata \ - --output-path "${{ runner.temp }}/outputs" - - - name: Read regression report - id: read-analysis - uses: juliangruber/read-file-action@v1 - with: - path: ${{ runner.temp }}/outputs/report.html - - - name: Post report to PR - uses: peter-evans/create-or-update-comment@v3 - with: - issue-number: ${{ needs.compute-metadata.outputs.github-event-number }} - edit-mode: append - body: ${{ steps.read-analysis.outputs.content }} - - - name: Upload regression report to artifacts - uses: actions/upload-artifact@v3 - with: - name: capture-artifacts - path: ${{ runner.temp }}/outputs/* - - - name: Check status, cancelled - if: ${{ cancelled() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='failure' \ - -f description='Analyze experimental results from Regression Detector cancelled.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, success - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Analyze experimental results from Regression Detector succeeded.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, failure - if: ${{ failure() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Analyze experimental results from Regression Detector failed.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24d78bfae85e0..1219da7a73d61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,16 +4,10 @@ on: pull_request: merge_group: types: [checks_requested] - push: - branches: - - master concurrency: - # For pull requests, cancel running workflows, for master, run all - # - # `github.event.number` exists for pull requests, otherwise fall back to SHA - # for master - group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue + group: ${{ github.workflow }}-${{ github.event.number || github.event.merge_group.base_sha }} cancel-in-progress: true env: @@ -33,63 +27,11 @@ env: jobs: changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - source: ${{ steps.filter.outputs.source }} - dependencies: ${{ steps.filter.outputs.dependencies }} - internal_events: ${{ steps.filter.outputs.internal_events }} - cue: ${{ steps.filter.outputs.cue }} - component_docs: ${{ steps.filter.outputs.component_docs }} - markdown: ${{ steps.filter.outputs.markdown }} - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source: - - ".github/workflows/test.yml" - - ".cargo/**" - - "benches/**" - - "lib/**" - - "proto/**" - - "scripts/**" - - "src/**" - - "tests/**" - - "build.rs" - - "Cargo.lock" - - "Cargo.toml" - - "Makefile" - - "rust-toolchain.toml" - - "vdev/**" - deny: - - 'deny.toml' - - "vdev/**" - dependencies: - - ".cargo/**" - - 'Cargo.toml' - - 'Cargo.lock' - - 'rust-toolchain.toml' - - '.github/workflows/pr.yml' - - 'Makefile' - - 'scripts/cross/**' - - "vdev/**" - cue: - - 'website/cue/**' - - "vdev" - component_docs: - - 'scripts/generate-component-docs.rb' - - "vdev/**" - markdown: - - '**/**.md' - - "vdev/**" - internal_events: - - 'src/internal_events/**' - - "vdev/**" - docker: - - 'distribution/docker/**' - - "vdev/**" + uses: ./.github/workflows/changes.yml + secrets: inherit + with: + base_ref: ${{ github.event.merge_group.base_ref || github.event.pull_request.base.ref }} + head_ref: ${{ github.event.merge_group.head_ref || github.event.pull_request.head.ref }} # Remove this once https://github.com/vectordotdev/vector/issues/3771 is closed. # Then, modify the `cross-linux` job to run `test` instead of `build`. @@ -123,153 +65,6 @@ jobs: run: scripts/upload-test-results.sh if: always() - test-cli: - name: CLI - Linux - runs-on: [linux, ubuntu-20.04-8core] - needs: changes - env: - CARGO_INCREMENTAL: 0 - if: ${{ needs.changes.outputs.source == 'true' }} - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Cache Cargo registry + index - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh - - run: echo "::add-matcher::.github/matchers/rust.json" - - run: make test-cli - - name: Upload test results - run: scripts/upload-test-results.sh - if: always() - - test-misc: - name: Miscellaneous - Linux - runs-on: [linux, ubuntu-20.04-8core] - needs: changes - env: - CARGO_INCREMENTAL: 0 - if: ${{ needs.changes.outputs.source == 'true' }} - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Cache Cargo registry + index - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh - - run: echo "::add-matcher::.github/matchers/rust.json" - - run: make test-behavior - - run: make check-examples - - run: make test-docs - - cross-linux: - name: Cross - ${{ matrix.target }} - runs-on: [linux, ubuntu-20.04-8core] - needs: changes - env: - CARGO_INCREMENTAL: 0 - strategy: - matrix: - target: - - x86_64-unknown-linux-gnu - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-gnu - - aarch64-unknown-linux-musl - - armv7-unknown-linux-gnueabihf - - armv7-unknown-linux-musleabihf - - # We run cross checks when dependencies change to ensure they still build. - # This helps us avoid adopting dependencies that aren't compatible with other architectures. - if: ${{ needs.changes.outputs.dependencies == 'true' }} - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Cache Cargo registry + index - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - run: echo "::add-matcher::.github/matchers/rust.json" - - run: 'cargo install cross --version 0.2.4 --force --locked' - # Why is this build, not check? Because we need to make sure the linking phase works. - # aarch64 and musl in particular are notoriously hard to link. - # While it may be tempting to slot a `check` in here for quickness, please don't. - - run: make cross-build-${{ matrix.target }} - - uses: actions/upload-artifact@v3 - with: - name: "vector-debug-${{ matrix.target }}" - path: "./target/${{ matrix.target }}/debug/vector" - - cross-linux-check: - if: ${{ needs.changes.outputs.dependencies == 'true' }} - runs-on: ubuntu-20.04 - name: Cross - Linux - needs: cross-linux - steps: - - name: Check cross matrix status - if: ${{ needs.cross-linux.result != 'success' }} - run: exit 1 - - test-mac: - name: Unit - Mac - # Full CI suites for this platform were only recently introduced. - # Some failures are permitted until we can properly correct them. - continue-on-error: true - runs-on: macos-11 - needs: changes - env: - CARGO_INCREMENTAL: 0 - if: ${{ needs.changes.outputs.source == 'true' }} - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - name: Cache Cargo registry + index - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - run: bash scripts/environment/bootstrap-macos-10.sh - - run: bash scripts/environment/prepare.sh - - run: echo "::add-matcher::.github/matchers/rust.json" - - run: make test - - run: make test-behavior - - test-windows: - name: Unit - Windows - runs-on: [windows, windows-2019-8core] - needs: changes - if: ${{ needs.changes.outputs.source == 'true' }} - steps: - - uses: actions/checkout@v3 - - run: .\scripts\environment\bootstrap-windows-2019.ps1 - - run: make test - test-vrl: name: VRL - Linux continue-on-error: true @@ -282,29 +77,6 @@ jobs: - run: bash scripts/environment/prepare.sh - run: cargo vdev test-vrl - check-component-features: - name: Component Features - Linux - runs-on: [linux, ubuntu-20.04-8core] - needs: changes - if: ${{ needs.changes.outputs.source == 'true' }} - steps: - - uses: actions/checkout@v3 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: bash scripts/environment/prepare.sh - - run: echo "::add-matcher::.github/matchers/rust.json" - - run: make check-component-features - - check-msrv: - name: Check minimum supported Rust version - runs-on: [linux, ubuntu-20.04-8core] - needs: changes - if: ${{ needs.changes.outputs.source == 'true' }} - steps: - - uses: actions/checkout@v3 - - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh - - run: cargo install cargo-msrv --version 0.15.1 - - run: cargo msrv verify - checks: name: Checks runs-on: [linux, ubuntu-20.04-8core] @@ -367,24 +139,14 @@ jobs: path: "/tmp/vector-config-schema.json" if: success() || failure() - master-failure: - name: master-failure - if: failure() && github.ref == 'refs/heads/master' + all-checks: + name: Test Suite + runs-on: ubuntu-20.04 needs: - - changes - - cross-linux - - test-misc - - test-linux - - test-mac - - test-windows - - test-vrl - - check-component-features - checks - runs-on: ubuntu-20.04 + - test-vrl + - test-linux steps: - - name: Discord notification - env: - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - uses: Ilshidur/action-discord@0.3.2 - with: - args: "Master tests failed: " + - name: validate + run: echo "OK" + diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml new file mode 100644 index 0000000000000..d922f05da370b --- /dev/null +++ b/.github/workflows/unit_mac.yml @@ -0,0 +1,65 @@ +name: Unit - Mac + +on: + workflow_call: + +jobs: + unit-mac: + # Full CI suites for this platform were only recently introduced. + # Some failures are permitted until we can properly correct them. + continue-on-error: true + runs-on: macos-11 + env: + CARGO_INCREMENTAL: 0 + steps: + + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Unit - Mac + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - uses: actions/cache@v3 + name: Cache Cargo registry + index + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - run: bash scripts/environment/bootstrap-macos-10.sh + - run: bash scripts/environment/prepare.sh + - run: echo "::add-matcher::.github/matchers/rust.json" + - run: make test + - run: make test-behavior + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@1.1.6 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Unit - Mac + status: ${{ job.status }} diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml new file mode 100644 index 0000000000000..1971670c5dbe6 --- /dev/null +++ b/.github/workflows/unit_windows.yml @@ -0,0 +1,53 @@ +name: Unit - Windows + +on: + workflow_call: + +jobs: + + test-windows: + runs-on: [windows, windows-2019-8core] + steps: + - name: Validate issue comment + if: github.event_name == 'issue_comment' + uses: tspascoal/get-user-teams-membership@v2 + with: + username: ${{ github.actor }} + team: 'Vector' + GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + + - name: (PR comment) Get PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: (PR comment) Set latest commit status as pending + if: ${{ github.event_name == 'issue_comment' }} + uses: myrotvorets/set-commit-status-action@1.1.6 + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Unit - Windows + status: pending + + - name: (PR comment) Checkout PR branch + if: ${{ github.event_name == 'issue_comment' }} + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} + + - name: Checkout branch + if: ${{ github.event_name != 'issue_comment' }} + uses: actions/checkout@v3 + + - run: .\scripts\environment\bootstrap-windows-2019.ps1 + - run: make test + + - name: (PR comment) Set latest commit status as ${{ job.status }} + uses: myrotvorets/set-commit-status-action@1.1.6 + if: always() && github.event_name == 'issue_comment' + with: + sha: ${{ steps.comment-branch.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + context: Unit - Windows + status: ${{ job.status }} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index cebda9652551b..bc87acb59fec1 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -67,6 +67,13 @@ To merge a new source, sink, or transform, you need to: see some [example of instrumentation in existing integrations](https://github.com/vectordotdev/vector/tree/master/src/internal_events). - [ ] Add documentation. You can see [examples in the `docs` directory](https://github.com/vectordotdev/vector/blob/master/docs). +When adding new integration tests, the following changes are needed in the github workflows: + +- in `.github/workflows/integration.yml`, add another entry in the matrix definition for the new integration. +- in `.github/workflows/integration-comment.yml`, add another entry in the matrix definition for the new integration. +- in `.github/workflows/changes.yml`, add a new filter definition for files changed, and update the `changes` job +outputs to reference the filter, and finally update the outputs of `workflow_call` to include the new filter. + ## Workflow ### Git Branches