workflows: collect additional metadata from GitHub and TC for EngFlow

We collect the following information from either GitHub Actions or TeamCity depending on where the job is being run: * Branch (PR number or build branch) * Job name * Run ID (+ attempt for GitHub Actions) On the GitHub Actions side, we plumb some of this into `--bes_keywords` for EngFlow. We also collect this data from environment variables and add it to the CSV report. Part of: DEVINF-1028 Epic: CRDB-8308 Release note: None
cockroachdb · Mar 12, 2024 · f76cc8c · f76cc8c
1 parent 3042d88
commit f76cc8c
Show file tree

Hide file tree

Showing 9 changed files with 108 additions and 33 deletions.
diff --git a/.github/workflows/github-actions-essential-ci.yml b/.github/workflows/github-actions-essential-ci.yml
@@ -4,15 +4,17 @@ on:
     types: [ opened, reopened, synchronize, edited ]
     branches: [ master ]
 concurrency:
-  group: ${{ github.event.pull_request.number }}
+  group: ${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 jobs:
   EXPERIMENTAL_acceptance:
     runs-on: [self-hosted, basic_big_runner_group]
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: run acceptance tests
@@ -33,7 +35,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - name: check generated code
         run: ./build/github/check-generated-code.sh
@@ -45,7 +49,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: run docker tests
@@ -62,12 +68,14 @@ jobs:
       - uses: actions/checkout@v4
         with:
           path: cockroach
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - uses: actions/checkout@v4
         with:
           path: examples-orms
           repository: cockroachdb/examples-orms
           ref: 876b2d52ae2b63aa9cc1741c8d189ff0b66ab0d7
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./cockroach/build/github/get-engflow-keys.sh
       - name: run tests
         run: ./cockroach/build/github/examples-orms.sh
@@ -79,8 +87,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
           submodules: true
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: run lint tests
@@ -96,8 +106,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
           submodules: true
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - name: run local roachtests
         run: ./build/github/local-roachtest.sh crosslinux
@@ -114,8 +126,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
           submodules: true
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - name: run local roachtests
         run: ./build/github/local-roachtest.sh crosslinuxfips
@@ -132,11 +146,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: build
-        run: bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-short //pkg/cmd/cockroach-sql //pkg/cmd/cockroach-oss //pkg/cmd/roachprod //pkg/cmd/workload //pkg/cmd/dev //c-deps:libgeos --config crossmacos --jobs 100 --remote_download_minimal --bes_keywords=github_pr_number=${{ github.event.pull_request.number }} --bes_keywords build-macos-amd64 --build_event_binary_file=bes.bin $(./build/github/engflow-args.sh)
+        run: bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-short //pkg/cmd/cockroach-sql //pkg/cmd/cockroach-oss //pkg/cmd/roachprod //pkg/cmd/workload //pkg/cmd/dev //c-deps:libgeos --config crossmacos --jobs 100 --remote_download_minimal --build_event_binary_file=bes.bin $(./build/github/engflow-args.sh)
       - name: upload build results
         run: ./build/github/summarize-build.sh bes.bin
         if: always()
@@ -148,11 +164,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: build
-        run: bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-short //pkg/cmd/cockroach-sql //pkg/cmd/cockroach-oss //pkg/cmd/roachprod //pkg/cmd/workload //pkg/cmd/dev //c-deps:libgeos --config crossmacosarm --jobs 100 --remote_download_minimal --bes_keywords=github_pr_number=${{ github.event.pull_request.number }} --bes_keywords build-macos-arm64 --build_event_binary_file=bes.bin $(./build/github/engflow-args.sh)
+        run: bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-short //pkg/cmd/cockroach-sql //pkg/cmd/cockroach-oss //pkg/cmd/roachprod //pkg/cmd/workload //pkg/cmd/dev //c-deps:libgeos --config crossmacosarm --jobs 100 --remote_download_minimal --build_event_binary_file=bes.bin $(./build/github/engflow-args.sh)
       - name: upload build results
         run: ./build/github/summarize-build.sh bes.bin
         if: always()
@@ -166,11 +184,13 @@ jobs:
         with:
           # By default, checkout merges the PR into the current master.
           # Instead, we want to check out the PR as is.
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: run tests
-        run: bazel test //pkg:all_tests //pkg/ui:lint //pkg/ui:test --config crosslinux --jobs 300 --remote_download_minimal --bes_keywords=github_pr_number=${{ github.event.pull_request.number }} --bes_keywords ci-unit-test --config=use_ci_timeouts --build_event_binary_file=bes.bin $(./build/github/engflow-args.sh)
+        run: bazel test //pkg:all_tests //pkg/ui:lint //pkg/ui:test --config crosslinux --jobs 300 --remote_download_minimal --bes_keywords ci-unit-test --config=use_ci_timeouts --build_event_binary_file=bes.bin $(./build/github/engflow-args.sh)
       - name: upload test results
         run: ./build/github/summarize-build.sh bes.bin
         if: always()
@@ -182,11 +202,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
       - run: ./build/github/get-engflow-keys.sh
       - run: ./build/github/prepare-summarize-build.sh
       - name: build
-        run: bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-short //pkg/cmd/cockroach-sql //pkg/cmd/cockroach-oss --config crosswindows --jobs 100 --remote_download_minimal --bes_keywords=github_pr_number=${{ github.event.pull_request.number }} --bes_keywords build-windows --build_event_binary_file=bes.bin --enable_runfiles $(./build/github/engflow-args.sh)
+        run: bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-short //pkg/cmd/cockroach-sql //pkg/cmd/cockroach-oss --config crosswindows --jobs 100 --remote_download_minimal --build_event_binary_file=bes.bin --enable_runfiles $(./build/github/engflow-args.sh)
       - name: upload build results
         run: ./build/github/summarize-build.sh bes.bin
         if: always()

diff --git a/build/github/acceptance-test.sh b/build/github/acceptance-test.sh
@@ -15,7 +15,6 @@ bazel test //pkg/acceptance:acceptance_test \
   --remote_download_minimal \
   "--sandbox_writable_path=$ARTIFACTSDIR" \
   "--test_tmpdir=$ARTIFACTSDIR" \
-  --bes_keywords acceptance \
   --test_arg=-l="$ARTIFACTSDIR" \
   --test_arg=-b=$COCKROACH \
   --test_env=TZ=America/New_York \

diff --git a/build/github/check-generated-code.sh b/build/github/check-generated-code.sh
@@ -25,7 +25,7 @@ if ! (./build/bazelutil/check.sh &> artifacts/check-out.log || (cat artifacts/ch
 fi
 rm artifacts/check-out.log
 
-ENGFLOW_ARGS="--config crosslinux --jobs 100 $(./build/github/engflow-args.sh) --remote_download_minimal --bes_keywords check-generated-code"
+ENGFLOW_ARGS="--config crosslinux --jobs 100 $(./build/github/engflow-args.sh) --remote_download_minimal"
 
 EXTRA_BAZEL_ARGS="$ENGFLOW_ARGS" \
     COCKROACH_BAZEL_FORCE_GENERATE=1 \

diff --git a/build/github/engflow-args.sh b/build/github/engflow-args.sh
@@ -3,4 +3,16 @@
 # remote execution arguments to the invocation. You must call get-engflow-keys.sh
 # before this.
 
-echo '--config engflowpublic --tls_client_certificate=/home/agent/engflow.crt --tls_client_key=/home/agent/engflow.key'
+ARGS='--config engflowpublic --tls_client_certificate=/home/agent/engflow.crt --tls_client_key=/home/agent/engflow.key'
+
+if [ ! -z "$GITHUB_ACTIONS_BRANCH" ]
+then
+    ARGS="$ARGS --bes_keywords branch=$GITHUB_ACTIONS_BRANCH"
+fi
+
+if [ ! -z "$GITHUB_JOB" ]
+then
+    ARGS="$ARGS --bes_keywords job=${GITHUB_JOB#EXPERIMENTAL_}"
+fi
+
+echo "$ARGS"
diff --git a/build/github/lint.sh b/build/github/lint.sh
@@ -7,13 +7,11 @@ WORKSPACE=$(bazel info workspace)
 # GCAssert and unused need generated files in the workspace to work properly.
 bazel run //pkg/gen:code \
     --config crosslinux --jobs 100 \
-    --remote_download_minimal $(./build/github/engflow-args.sh) \
-    --bes_keywords=lint
+    --remote_download_minimal $(./build/github/engflow-args.sh)
 bazel run //pkg/cmd/generate-cgo:generate-cgo \
     --run_under="cd $WORKSPACE && " \
     --config crosslinux --jobs 100 \
-    --remote_download_minimal $(./build/github/engflow-args.sh) \
-    --bes_keywords=lint
+    --remote_download_minimal $(./build/github/engflow-args.sh)
 
 bazel test \
   //pkg/testutils/lint:lint_test \
@@ -27,5 +25,4 @@ bazel test \
   --test_timeout=1800 \
   --build_event_binary_file=bes.bin \
   --jobs 100 \
-  --remote_download_minimal $(./build/github/engflow-args.sh) \
-  --bes_keywords=lint
+  --remote_download_minimal $(./build/github/engflow-args.sh)
diff --git a/build/github/local-roachtest.sh b/build/github/local-roachtest.sh
@@ -22,7 +22,7 @@ export COCKROACH_DEV_LICENSE=$(gcloud secrets versions access 1 --secret=cockroa
 set -x
 
 bazel build --config=$CROSSCONFIG $(./build/github/engflow-args.sh) \
-      --bes_keywords=local-roachtest --jobs 100 \
+      --jobs 100 \
       //pkg/cmd/cockroach-short \
       //pkg/cmd/roachtest \
       //pkg/cmd/roachprod \

diff --git a/build/teamcity/cockroach/nightlies/stress_engflow_impl.sh b/build/teamcity/cockroach/nightlies/stress_engflow_impl.sh
@@ -27,6 +27,8 @@ bazel test //pkg:all_tests $ENGFLOW_FLAGS --remote_download_minimal \
       --profile=artifacts/profile.json.gz \
       ${EXTRA_TEST_ARGS:+$EXTRA_TEST_ARGS} \
       $BES_KEYWORDS_ARGS \
+      --bes_keywords "branch=${TC_BUILD_BRANCH#refs/heads/}" \
+      --bes_keywords nightly_stress \
     || status=$?
 
 # Upload results to GitHub.

diff --git a/pkg/build/engflow/engflow.go b/pkg/build/engflow/engflow.go
@@ -77,6 +77,15 @@ type JsonReport struct {
 	ExitCode       int32                       `json:"exit_code"`
 	ExitCodeName   string                      `json:"exit_code_name"`
 	ResultsByLabel map[string][]JsonTestResult `json:"results_by_label"`
+
+	GitHubRunId      int64  `json:"github_run_id,omitempty"`
+	GitHubJob        string `json:"github_job,omitempty"`
+	GitHubRunAttempt int64  `json:"github_run_attempt,omitempty"`
+
+	TeamCityBuildTypeId string `json:"teamcity_buildtype_id,omitempty"`
+	TeamCityBuildId     int64  `json:"teamcity_build_id,omitempty"`
+
+	Branch string `json:"branch,omitempty"`
 }
 
 type JsonTestResult struct {
@@ -353,15 +362,29 @@ func stringToMillis(s string) (int64, error) {
 // be parsed or was not fetched, then the report will be missing those test
 // results, but everything else will be present. If errs is empty, then the
 // report is complete.
+//
+// In addition to the passed-in arguments, we also consult the following
+// environment variables for supplemental data: GITHUB_RUN_ID, GITHUB_JOB,
+// GITHUB_RUN_ATTEMPT, TC_BUILDTYPE_ID, TC_BUILD_ID, GITHUB_ACTIONS_BRANCH and
+// TC_BUILD_BRANCH. These are environment variables that either GitHub sets for us (see
+// https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables),
+// we set in TeamCity, or we set as part of the GitHub workflow definition (see
+// github-actions-essential-ci.yml).
 func ConstructJSONReport(invocation *InvocationInfo, serverName string) (JsonReport, []error) {
 	ret := JsonReport{
-		Server:         serverName,
-		InvocationId:   invocation.InvocationId,
-		StartedAt:      timeMillisToString(invocation.StartedTimeMillis),
-		FinishedAt:     timeMillisToString(invocation.FinishTimeMillis),
-		ExitCode:       invocation.ExitCode,
-		ExitCodeName:   invocation.ExitCodeName,
-		ResultsByLabel: make(map[string][]JsonTestResult),
+		Server:              serverName,
+		InvocationId:        invocation.InvocationId,
+		StartedAt:           timeMillisToString(invocation.StartedTimeMillis),
+		FinishedAt:          timeMillisToString(invocation.FinishTimeMillis),
+		ExitCode:            invocation.ExitCode,
+		ExitCodeName:        invocation.ExitCodeName,
+		ResultsByLabel:      make(map[string][]JsonTestResult),
+		GitHubRunId:         tryParseInt(os.Getenv("GITHUB_RUN_ID")),
+		GitHubJob:           os.Getenv("GITHUB_JOB"),
+		GitHubRunAttempt:    tryParseInt(os.Getenv("GITHUB_RUN_ATTEMPT")),
+		TeamCityBuildTypeId: os.Getenv("TC_BUILDTYPE_ID"),
+		TeamCityBuildId:     tryParseInt(os.Getenv("TC_BUILD_ID")),
+		Branch:              getBranch(),
 	}
 	var errs []error
 
@@ -406,3 +429,22 @@ func ConstructJSONReport(invocation *InvocationInfo, serverName string) (JsonRep
 
 	return ret, errs
 }
+
+func tryParseInt(s string) int64 {
+	if s == "" {
+		return 0
+	}
+	i, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0
+	}
+	return i
+}
+
+func getBranch() string {
+	b := os.Getenv("GITHUB_ACTIONS_BRANCH")
+	if b != "" {
+		return b
+	}
+	return strings.TrimPrefix(os.Getenv("TC_BUILD_BRANCH"), "refs/heads/")
+}
diff --git a/pkg/testutils/lint/lint_test.go b/pkg/testutils/lint/lint_test.go
@@ -548,6 +548,7 @@ func TestLint(t *testing.T) {
 					":!testutils/backup.go",                  // For BACKUP_TESTING_BUCKET
 					":!compose/compose_test.go",              // For PATH.
 					":!testutils/skip/skip.go",               // For REMOTE_EXEC.
+					":!build/engflow/engflow.go",             // For GITHUB_ACTIONS_BRANCH, etc.
 				},
 			},
 		} {