release-23.1: build,release: provide way to inject build tag override, use in MAPB

build/bazelutil/stamp.sh

@@ -10,6 +10,7 @@
 #  target-triple: defaults to the value of `cc -dumpmachine`
 #  build-channel: defaults to `unknown`, but can be `official-binary`
 #  build-type: defaults to `development`, but can be `release`
+#  build-tag: will default to an appropriate value if not passed in, but can be overridden
 
 set -euo pipefail
 
@@ -56,7 +57,16 @@ else
     CRASH_REPORT_ENV="development"
 fi
 
-BUILD_REV=$(git rev-parse HEAD)
+# Handle build-tag.
+if [ -z "${1+x}" ]
+then
+    BUILD_TAG=
+else
+    BUILD_TAG="$1"
+    shift 1
+fi
+
+BUILD_REV=$(git describe --match="" --always --abbrev=40 --dirty)
 BUILD_UTCTIME=$(date -u '+%Y/%m/%d %H:%M:%S')
 
 
@@ -69,6 +79,7 @@ BUILD_UTCTIME=$(date -u '+%Y/%m/%d %H:%M:%S')
 # * https://github.com/bazelbuild/rules_go/blob/master/go/core.rst#defines-and-stamping
 cat <<EOF
 STABLE_BUILD_CHANNEL ${BUILD_CHANNEL-}
+STABLE_BUILD_TAG ${BUILD_TAG-}
 STABLE_BUILD_TARGET_TRIPLE ${TARGET_TRIPLE-}
 STABLE_BUILD_TYPE ${BUILD_TYPE-}
 STABLE_CRASH_REPORT_ENV ${CRASH_REPORT_ENV-}

build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh

@@ -52,14 +52,14 @@ git tag "${build_name}"
 tc_end_block "Tag the release"
 
 tc_start_block "Compile and publish artifacts"
-BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH=$build_name -e build_name=$build_name -e gcs_credentials -e gcs_bucket=$gcs_bucket" run_bazel << 'EOF'
 bazel build --config ci //pkg/cmd/publish-provisional-artifacts
 BAZEL_BIN=$(bazel info bazel-bin --config ci)
 export google_credentials="$gcs_credentials"
 source "build/teamcity-support.sh"  # For log_into_gcloud
 log_into_gcloud
 export GOOGLE_APPLICATION_CREDENTIALS="$PWD/.google-credentials.json"
-$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release --gcs-bucket="$gcs_bucket" --output-directory=artifacts
+$BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/publish-provisional-artifacts -provisional -release --gcs-bucket="$gcs_bucket" --output-directory=artifacts --build-tag-override="$build_name"
 EOF
 tc_end_block "Compile and publish artifacts"
 

pkg/build/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/build",
     visibility = ["//visibility:public"],
     x_defs = {
+        "github.com/cockroachdb/cockroach/pkg/build.buildTagOverride": "{STABLE_BUILD_TAG}",
         "github.com/cockroachdb/cockroach/pkg/build.cgoTargetTriple": "{STABLE_BUILD_TARGET_TRIPLE}",
         "github.com/cockroachdb/cockroach/pkg/build.channel": "{STABLE_BUILD_CHANNEL}",
         "github.com/cockroachdb/cockroach/pkg/build.rev": "{BUILD_REV}",

pkg/build/info.go

@@ -30,11 +30,12 @@ const TimeFormat = "2006/01/02 15:04:05"
 var (
 	// These variables are initialized by Bazel via the linker -X flag
 	// when compiling release binaries.
-	utcTime         string // Build time in UTC (year/month/day hour:min:sec)
-	rev             string // SHA-1 of this build (git rev-parse)
-	cgoCompiler     = cgoVersion()
-	cgoTargetTriple string
-	platform        = fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH)
+	utcTime          string // Build time in UTC (year/month/day hour:min:sec)
+	rev              string // SHA-1 of this build (git rev-parse)
+	buildTagOverride string
+	cgoCompiler      = cgoVersion()
+	cgoTargetTriple  string
+	platform         = fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH)
 	// Distribution is changed by the CCL init-time hook in non-APL builds.
 	Distribution = "OSS"
 	typ          string // Type of this build: <empty>, "development", or "release"
@@ -62,6 +63,9 @@ func SeemsOfficial() bool {
 }
 
 func computeBinaryVersion(versionTxt, revision string) string {
+	if buildTagOverride != "" {
+		return buildTagOverride
+	}
 	txt := strings.TrimSuffix(versionTxt, "\n")
 	v, err := version.Parse(txt)
 	if err != nil {

pkg/cmd/publish-provisional-artifacts/main.go

@@ -33,13 +33,15 @@ var provisionalReleasePrefixRE = regexp.MustCompile(`^provisional_[0-9]{12}_`)
 func main() {
 	var gcsBucket string
 	var outputDirectory string
+	var buildTagOverride string
 	var doProvisional bool
 	var isRelease bool
 	var doBless bool
 	flag.BoolVar(&isRelease, "release", false, "build in release mode instead of bleeding-edge mode")
 	flag.StringVar(&gcsBucket, "gcs-bucket", "", "GCS bucket")
 	flag.StringVar(&outputDirectory, "output-directory", "",
 		"Save local copies of uploaded release archives in this directory")
+	flag.StringVar(&buildTagOverride, "build-tag-override", "", "override the version from version.txt")
 	flag.BoolVar(&doProvisional, "provisional", false, "publish provisional binaries")
 	flag.BoolVar(&doBless, "bless", false, "bless provisional binaries")
 
@@ -82,24 +84,26 @@ func main() {
 	}
 
 	run(providers, runFlags{
-		doProvisional:   doProvisional,
-		doBless:         doBless,
-		isRelease:       isRelease,
-		branch:          branch,
-		pkgDir:          pkg,
-		sha:             string(bytes.TrimSpace(shaOut)),
-		outputDirectory: outputDirectory,
+		doProvisional:    doProvisional,
+		doBless:          doBless,
+		isRelease:        isRelease,
+		buildTagOverride: buildTagOverride,
+		branch:           branch,
+		pkgDir:           pkg,
+		sha:              string(bytes.TrimSpace(shaOut)),
+		outputDirectory:  outputDirectory,
 	}, release.ExecFn{})
 }
 
 type runFlags struct {
-	doProvisional   bool
-	doBless         bool
-	isRelease       bool
-	branch          string
-	sha             string
-	pkgDir          string
-	outputDirectory string
+	doProvisional    bool
+	doBless          bool
+	isRelease        bool
+	buildTagOverride string
+	branch           string
+	sha              string
+	pkgDir           string
+	outputDirectory  string
 }
 
 func run(providers []release.ObjectPutGetter, flags runFlags, execFn release.ExecFn) {
@@ -241,7 +245,9 @@ func buildCockroach(flags runFlags, o opts, execFn release.ExecFn) {
 	}
 	if flags.isRelease {
 		buildOpts.Release = true
-		buildOpts.BuildTag = o.VersionStr
+	}
+	if flags.buildTagOverride != "" {
+		buildOpts.BuildTag = flags.buildTagOverride
 	}
 
 	if err := release.MakeRelease(o.Platform, buildOpts, o.PkgDir); err != nil {

pkg/cmd/publish-provisional-artifacts/main_test.go

@@ -225,6 +225,67 @@ func TestProvisional(t *testing.T) {
 				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-arm64.tgz.sha256sum CONTENTS <sha256sum>",
 			},
 		},
+		{
+			name: `release-override-tag`,
+			flags: runFlags{
+				doProvisional:    true,
+				isRelease:        true,
+				branch:           `provisional_201901010101_v0.0.1-alpha`,
+				buildTagOverride: "injected-tag",
+			},
+			expectedCmds: []string{
+				"env=[] args=bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-sql //c-deps:libgeos " +
+					"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-binary release injected-tag' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase",
+				"env=[] args=bazel info bazel-bin -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxbase",
+				"env=[MALLOC_CONF=prof:true] args=./cockroach.linux-2.6.32-gnu-amd64 version",
+				"env=[] args=ldd ./cockroach.linux-2.6.32-gnu-amd64",
+				"env=[] args=bazel run @go_sdk//:bin/go -- tool nm ./cockroach.linux-2.6.32-gnu-amd64",
+				"env=[] args=bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-sql //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh x86_64-pc-linux-gnu official-fips-binary release injected-tag' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxfipsbase",
+				"env=[] args=bazel info bazel-bin -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxfipsbase",
+				"env=[MALLOC_CONF=prof:true] args=./cockroach.linux-2.6.32-gnu-amd64-fips version",
+				"env=[] args=ldd ./cockroach.linux-2.6.32-gnu-amd64-fips",
+				"env=[] args=bazel run @go_sdk//:bin/go -- tool nm ./cockroach.linux-2.6.32-gnu-amd64-fips",
+				"env=[] args=bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-sql //c-deps:libgeos " +
+					"'--workspace_status_command=./build/bazelutil/stamp.sh x86_64-apple-darwin19 official-binary release injected-tag' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase",
+				"env=[] args=bazel info bazel-bin -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosbase",
+				"env=[] args=bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-sql '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-apple-darwin21.2 official-binary release injected-tag' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosarmbase",
+				"env=[] args=bazel info bazel-bin -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crossmacosarmbase",
+				"env=[] args=bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-sql //c-deps:libgeos " +
+					"'--workspace_status_command=." +
+					"/build/bazelutil/stamp.sh x86_64-w64-mingw32 official-binary release injected-tag' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase",
+				"env=[] args=bazel info bazel-bin -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosswindowsbase",
+				"env=[] args=bazel build //pkg/cmd/cockroach //pkg/cmd/cockroach-sql //c-deps:libgeos '--workspace_status_command=./build/bazelutil/stamp.sh aarch64-unknown-linux-gnu official-binary release injected-tag' -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase",
+				"env=[] args=bazel info bazel-bin -c opt --config=ci --config=force_build_cdeps --config=with_ui --config=crosslinuxarmbase",
+			},
+			expectedGets: nil,
+			expectedPuts: []string{
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.linux-amd64.tgz " +
+					"CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.linux-amd64.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-amd64.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-amd64.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.linux-amd64-fips.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.linux-amd64-fips.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-amd64-fips.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-amd64-fips.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.darwin-10.9-amd64.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.darwin-10.9-amd64.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.darwin-10.9-amd64.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.darwin-10.9-amd64.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.darwin-11.0-arm64.unsigned.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.darwin-11.0-arm64.unsigned.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.darwin-11.0-arm64.unsigned.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.darwin-11.0-arm64.unsigned.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.windows-6.2-amd64.zip CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.windows-6.2-amd64.zip.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.windows-6.2-amd64.zip CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.windows-6.2-amd64.zip.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.linux-arm64.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-v0.0.1-alpha.linux-arm64.tgz.sha256sum CONTENTS <sha256sum>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-arm64.tgz CONTENTS <binary stuff>",
+				"gs://release-binaries-bucket/cockroach-sql-v0.0.1-alpha.linux-arm64.tgz.sha256sum CONTENTS <sha256sum>",
+			},
+		},
 		{
 			name: `edge`,
 			flags: runFlags{

pkg/release/build.go

@@ -31,7 +31,8 @@ import (
 type BuildOptions struct {
 	// True iff this is a release build.
 	Release bool
-	// BuildTag must be set if Release is set, and vice-versea.
+	// BuildTag overrides the build tag for a "Release" build.
+	// This can only be set for a Release build.
 	BuildTag string
 
 	// ExecFn.Run() is called to execute commands for this build.
@@ -163,17 +164,20 @@ func MakeRelease(platform Platform, opts BuildOptions, pkgDir string) error {
 		buildArgs = append(buildArgs, "//c-deps:libgeos")
 	}
 	targetTriple := TargetTripleFromPlatform(platform)
+	var stampCommand string
 	if opts.Release {
 		if opts.BuildTag == "" {
-			return errors.Newf("must set BuildTag if Release is set")
+			stampCommand = fmt.Sprintf("--workspace_status_command=./build/bazelutil/stamp.sh %s %s release", targetTriple, opts.Channel)
+		} else {
+			stampCommand = fmt.Sprintf("--workspace_status_command=./build/bazelutil/stamp.sh %s %s release %s", targetTriple, opts.Channel, opts.BuildTag)
 		}
-		buildArgs = append(buildArgs, fmt.Sprintf("--workspace_status_command=./build/bazelutil/stamp.sh %s %s release", targetTriple, opts.Channel))
 	} else {
 		if opts.BuildTag != "" {
-			return errors.Newf("cannot set BuildTag if Release is not set")
+			return errors.Newf("BuildTag cannot be set for non-Release builds")
 		}
-		buildArgs = append(buildArgs, fmt.Sprintf("--workspace_status_command=./build/bazelutil/stamp.sh %s %s", targetTriple, opts.Channel))
+		stampCommand = fmt.Sprintf("--workspace_status_command=./build/bazelutil/stamp.sh %s %s", targetTriple, opts.Channel)
 	}
+	buildArgs = append(buildArgs, stampCommand)
 	configs := []string{"-c", "opt", "--config=ci", "--config=force_build_cdeps", "--config=with_ui", fmt.Sprintf("--config=%s", CrossConfigFromPlatform(platform))}
 	buildArgs = append(buildArgs, configs...)
 	cmd := exec.Command("bazel", buildArgs...)