Skip to content

Commit

Permalink
chore(ci): Move most CI checks to merge queue (vectordotdev#17340)
Browse files Browse the repository at this point in the history 
Introduces a number of changes to our project's GH workflows:
- There is a more limited set of tests that run on each PR commit,
and the rest were moved to the merge queue.
- Most workflows can be triggered manually by a team member
though a comment on a PR.
- Integration tests are run on PR commits if they touch the files
specific to that component.
  • Loading branch information
@neuronull
neuronull authored May 22, 2023
1 parent 618379a commit 060399a
Show file tree
Hide file tree
Showing 21 changed files with 2,308 additions and 1,101 deletions.
333 changes: 333 additions & 0 deletions .github/workflows/changes.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,333 @@
# This workflow identifies changes between the base and the head ref, for use in
# other workflows to decide if they should be executed.

name: Identify Changes

on:
workflow_call:
# These inputs allow the filter action to be able to access the correct refs for
# comparison in changes detection, it is required as this is called from the
# merge_group context.
inputs:
base_ref:
required: true
type: string
head_ref:
required: true
type: string
outputs:
source:
value: ${{ jobs.changes.outputs.source }}
dependencies:
value: ${{ jobs.changes.outputs.dependencies }}
internal_events:
value: ${{ jobs.changes.outputs.internal_events }}
cue:
value: ${{ jobs.changes.outputs.cue }}
component_docs:
value: ${{ jobs.changes.outputs.component_docs }}
markdown:
value: ${{ jobs.changes.outputs.markdown }}
install:
value: ${{ jobs.changes.outputs.install }}
k8s:
value: ${{ jobs.changes.outputs.k8s }}
all-int:
value: ${{ jobs.changes.outputs.all-int }}
amqp:
value: ${{ jobs.changes.outputs.amqp }}
appsignal:
value: ${{ jobs.changes.outputs.appsignal }}
aws:
value: ${{ jobs.changes.outputs.aws }}
axiom:
value: ${{ jobs.changes.outputs.axiom }}
azure:
value: ${{ jobs.changes.outputs.azure }}
clickhouse:
value: ${{ jobs.changes.outputs.clickhouse }}
databend:
value: ${{ jobs.changes.outputs.databend }}
datadog:
value: ${{ jobs.changes.outputs.datadog }}
dnstap:
value: ${{ jobs.changes.outputs.dnstap }}
docker-logs:
value: ${{ jobs.changes.outputs.docker-logs }}
elasticsearch:
value: ${{ jobs.changes.outputs.elasticsearch }}
eventstoredb:
value: ${{ jobs.changes.outputs.eventstoredb }}
fluent:
value: ${{ jobs.changes.outputs.fluent }}
gcp:
value: ${{ jobs.changes.outputs.gcp }}
humio:
value: ${{ jobs.changes.outputs.humio }}
http-client:
value: ${{ jobs.changes.outputs.http-client }}
influxdb:
value: ${{ jobs.changes.outputs.influxdb }}
kafka:
value: ${{ jobs.changes.outputs.kafka }}
logstash:
value: ${{ jobs.changes.outputs.logstash }}
loki:
value: ${{ jobs.changes.outputs.loki }}
mongodb:
value: ${{ jobs.changes.outputs.mongodb }}
nats:
value: ${{ jobs.changes.outputs.nats }}
nginx:
value: ${{ jobs.changes.outputs.nginx }}
opentelemetry:
value: ${{ jobs.changes.outputs.opentelemetry }}
postgres:
value: ${{ jobs.changes.outputs.postgres }}
prometheus:
value: ${{ jobs.changes.outputs.prometheus }}
pulsar:
value: ${{ jobs.changes.outputs.pulsar }}
redis:
value: ${{ jobs.changes.outputs.redis }}
splunk:
value: ${{ jobs.changes.outputs.splunk }}
webhdfs:
value: ${{ jobs.changes.outputs.webhdfs }}

jobs:
changes:
runs-on: ubuntu-20.04
# Set job outputs to values from filter step
outputs:
# General source code
source: ${{ steps.filter.outputs.source }}
dependencies: ${{ steps.filter.outputs.dependencies }}
internal_events: ${{ steps.filter.outputs.internal_events }}
cue: ${{ steps.filter.outputs.cue }}
component_docs: ${{ steps.filter.outputs.component_docs }}
markdown: ${{ steps.filter.outputs.markdown }}
install: ${{ steps.filter.outputs.install }}
# K8s
k8s: ${{ steps.filter.outputs.k8s }}
# Integrations
all-int: ${{ steps.filter.outputs.all-int }}
amqp: ${{ steps.filter.outputs.amqp }}
appsignal: ${{ steps.filter.outputs.appsignal}}
aws: ${{ steps.filter.outputs.aws }}
axiom: ${{ steps.filter.outputs.axiom }}
azure: ${{ steps.filter.outputs.azure }}
clickhouse: ${{ steps.filter.outputs.clickhouse }}
databend: ${{ steps.filter.outputs.databend }}
datadog: ${{ steps.filter.outputs.datadog }}
dnstap: ${{ steps.filter.outputs.dnstap }}
docker-logs: ${{ steps.filter.outputs.docker-logs }}
elasticsearch: ${{ steps.filter.outputs.elasticsearch }}
eventstoredb: ${{ steps.filter.outputs.eventstoredb }}
fluent: ${{ steps.filter.outputs.fluent }}
gcp: ${{ steps.filter.outputs.gcp }}
humio: ${{ steps.filter.outputs.humio }}
http-client: ${{ steps.filter.outputs.http-client }}
influxdb: ${{ steps.filter.outputs.influxdb }}
kafka: ${{ steps.filter.outputs.kafka }}
logstash: ${{ steps.filter.outputs.logstash }}
loki: ${{ steps.filter.outputs.loki }}
mongodb: ${{ steps.filter.outputs.mongodb }}
nats: ${{ steps.filter.outputs.nats }}
nginx: ${{ steps.filter.outputs.nginx }}
opentelemetry: ${{ steps.filter.outputs.opentelemetry }}
postgres: ${{ steps.filter.outputs.postgres }}
prometheus: ${{ steps.filter.outputs.prometheus }}
pulsar: ${{ steps.filter.outputs.pulsar }}
redis: ${{ steps.filter.outputs.redis }}
splunk: ${{ steps.filter.outputs.splunk }}
webhdfs: ${{ steps.filter.outputs.webhdfs }}
steps:
- uses: actions/checkout@v3

- uses: dorny/paths-filter@v2
id: filter
with:
base: ${{ inputs.base_ref }}
ref: ${{ inputs.head_ref }}
filters: |
source:
- ".github/workflows/test.yml"
- ".cargo/**"
- "benches/**"
- "lib/**"
- "proto/**"
- "scripts/**"
- "src/**"
- "tests/**"
- "build.rs"
- "Cargo.lock"
- "Cargo.toml"
- "Makefile"
- "rust-toolchain.toml"
- "vdev/**"
deny:
- 'deny.toml'
- "vdev/**"
dependencies:
- ".cargo/**"
- 'Cargo.toml'
- 'Cargo.lock'
- 'rust-toolchain.toml'
- '.github/workflows/pr.yml'
- 'Makefile'
- 'scripts/cross/**'
- "vdev/**"
cue:
- 'website/cue/**'
- "vdev"
component_docs:
- 'scripts/generate-component-docs.rb'
- "vdev/**"
markdown:
- '**/**.md'
- "vdev/**"
internal_events:
- 'src/internal_events/**'
- "vdev/**"
docker:
- 'distribution/docker/**'
- "vdev/**"
install:
- ".github/workflows/install-sh.yml"
- "distribution/install.sh"
k8s:
- "src/sources/kubernetes_logs/**"
all-int:
- "lib/vector-core/**"
amqp:
- "src/amqp.rs"
- "src/internal_events/amqp.rs"
- "src/sinks/amqp/**"
- "src/sources/amqp.rs"
- "src/sources/util/**"
- "src/sinks/util/**"
appsignal:
- "src/sinks/appsignal/**"
- "src/sinks/util/**"
aws:
- "src/aws_**"
- "src/internal_events/aws_**"
- "src/sources/aws_**"
- "src/sources/util/**"
- "src/sinks/aws_**"
- "src/sinks/util/**"
- "src/transforms/aws_**"
axiom:
- "src/sinks/axiom.rs"
- "src/sinks/util/**"
azure:
- "src/sinks/azure_**"
- "src/sinks/util/**"
clickhouse:
- "src/sinks/clickhouse/**"
- "src/sinks/util/**"
databend:
- "src/sinks/databend/**"
- "src/sinks/util/**"
datadog:
- "src/common/datadog.rs"
- "src/internal_events/datadog_*"
- "src/sources/datadog_agent/**"
- "src/sinks/datadog/**"
- "src/sinks/datadog_archives.rs"
- "src/sinks/util/**"
docker-logs:
- "src/docker.rs"
- "src/internal_events/docker_logs.rs"
- "src/sources/docker_logs/**"
- "src/sources/util/**"
elasticsearch:
- "src/sinks/elasticsearch/**"
- "src/sinks/util/**"
eventstoredb:
- "src/internal_events/eventstoredb_metrics.rs"
- "src/sources/eventstoredb_metrics/**"
- "src/sources/util/**"
fluent:
- "src/internal_events/fluent.rs"
- "src/sources/fluent/**"
- "src/sources/util/**"
gcp:
- "src/internal_events/gcp_pubsub.rs"
- "src/sources/gcp_pubsub.rs"
- "src/sources/util/**"
- "src/sinks/gcp/**"
- "src/sinks/util/**"
- "src/gcp.rs"
humio:
- "src/sinks/humio/**"
- "src/sinks/util/**"
http-client:
- "src/sinks/http-client/**"
influxdb:
- "src/internal_events/influxdb.rs"
- "src/sinks/influxdb/**"
- "src/sinks/util/**"
kafka:
- "src/internal_events/kafka.rs"
- "src/sinks/kafka/**"
- "src/sinks/util/**"
- "src/sources/kafka.rs"
- "src/sources/util/**"
- "src/kafka.rs"
logstash:
- "src/sources/logstash.rs"
- "src/sources/util/**"
loki:
- "src/internal_events/loki.rs"
- "src/sinks/loki/**"
- "src/sinks/util/**"
mongodb:
- "src/internal_events/mongodb_metrics.rs"
- "src/sources/mongodb_metrics/**"
- "src/sources/util/**"
nats:
- "src/internal_events/nats.rs"
- "src/sources/nats.rs"
- "src/sources/util/**"
- "src/sinks/nats.rs"
- "src/sinks/util/**"
- "src/nats.rs"
nginx:
- "src/internal_events/nginx_metrics.rs"
- "src/sources/nginx_metrics/**"
- "src/sources/util/**"
opentelemetry:
- "src/sources/opentelemetry/**"
- "src/sources/util/**"
postgres:
- "src/internal_events/postgresql_metrics.rs"
- "src/sources/postgresql_metrics.rs"
- "src/sources/util/**"
prometheus:
- "src/internal_events/prometheus.rs"
- "src/sources/prometheus/**"
- "src/sources/util/**"
- "src/sinks/prometheus/**"
- "src/sinks/util/**"
pulsar:
- "src/internal_events/pulsar.rs"
- "src/sinks/pulsar/**"
- "src/sinks/util/**"
redis:
- "src/internal_events/redis.rs"
- "src/sources/redis/**"
- "src/sources/util/**"
- "src/sinks/redis.rs"
- "src/sinks/util/**"
splunk:
- "src/internal_events/splunk_hec.rs"
- "src/sources/splunk_hec/**"
- "src/sources/util/**"
- "src/sinks/splunk_hec/**"
- "src/sinks/util/**"
webhdfs:
- "src/sinks/webhdfs/**"
- "src/sinks/util/**"
63 changes: 63 additions & 0 deletions .github/workflows/cli.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: CLI - Linux

on:
workflow_call:

jobs:
test-cli:
runs-on: [linux, ubuntu-20.04-8core]
env:
CARGO_INCREMENTAL: 0
steps:
- name: (PR comment) Get PR branch
if: ${{ github.event_name == 'issue_comment' }}
uses: xt0rted/pull-request-comment-branch@v1
id: comment-branch

- name: (PR comment) Set latest commit status as pending
if: ${{ github.event_name == 'issue_comment' }}
uses: myrotvorets/[email protected]
with:
sha: ${{ steps.comment-branch.outputs.head_sha }}
token: ${{ secrets.GITHUB_TOKEN }}
context: CLI - Linux
status: pending

- name: (PR comment) Checkout PR branch
if: ${{ github.event_name == 'issue_comment' }}
uses: actions/checkout@v3
with:
ref: ${{ steps.comment-branch.outputs.head_ref }}

- name: Checkout branch
if: ${{ github.event_name != 'issue_comment' }}
uses: actions/checkout@v3

- name: Cache Cargo registry + index
uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-
- run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
- run: bash scripts/environment/prepare.sh
- run: echo "::add-matcher::.github/matchers/rust.json"
- run: make test-cli
- name: Upload test results
run: scripts/upload-test-results.sh
if: always()

- name: (PR comment) Set latest commit status as ${{ job.status }}
uses: myrotvorets/[email protected]
if: always() && github.event_name == 'issue_comment'
with:
sha: ${{ steps.comment-branch.outputs.head_sha }}
token: ${{ secrets.GITHUB_TOKEN }}
context: CLI - Linux
status: ${{ job.status }}
Loading

0 comments on commit 060399a

Please sign in to comment.