From 2e5bfd62d166501437e14b10368fda9e448f4074 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 26 Dec 2023 15:46:35 +0800 Subject: [PATCH 1/2] feat(packages): support enterprise build profile for tidb - container images. - add plugins tarballs. - will not publish tiup for enterprise profile. Signed-off-by: wuhuizuo --- packages/README.md | 7 + packages/packages.yaml.tmpl | 197 +++++++++++++++++- .../scripts/build-package-artifacts.sh.tmpl | 14 +- packages/scripts/build-package-images.sh.tmpl | 4 +- 4 files changed, 210 insertions(+), 12 deletions(-) diff --git a/packages/README.md b/packages/README.md index 559df438..2728e0c6 100644 --- a/packages/README.md +++ b/packages/README.md @@ -10,6 +10,13 @@ Central declarative congfigurations for artifacts delivering. - [yq] - jq +## Profiles + +- `release`: community release profile. +- `debug`: test or debug profile. +- `enterprise`: enterprise release profile, it will not publish any tiup pkgs. +- `fips`: fips feature release without enterprise plugins. + ## For component binaries packages and container images Configuration template: [packages.yaml.tmpl](./packages.yaml.tmpl) diff --git a/packages/packages.yaml.tmpl b/packages/packages.yaml.tmpl index 043d4ccf..e96645c0 100644 --- a/packages/packages.yaml.tmpl +++ b/packages/packages.yaml.tmpl @@ -403,25 +403,57 @@ components: - if: {{ semver.CheckConstraint "< 6.1.0-0" .Release.version }} image: ghcr.io/pingcap-qe/cd/builders/tidb:v20231115-e1c4b43-go1.18 routers: - - description: From 6.1.0 + - description: From 7.1.0 # ref: https://github.com/Masterminds/semver#checking-version-constraints # fips profile only invalid in v6.5.x - if: {{ or (semver.CheckConstraint ">= 6.1.0-0" .Release.version) (and (semver.CheckConstraint "~6.5.0-0" .Release.version)) (eq "fips" .Release.profile) }} + if: {{ semver.CheckConstraint ">= 7.1.0-0" .Release.version }} os: [linux, darwin] arch: [amd64, arm64] - profile: [release, debug, fips] + profile: [release, debug, enterprise] steps: release: - script: | make server build_tools build_dumpling + enterprise: + - script: | + TIDB_EDITION=Enterprise make enterprise-prepare enterprise-server-build build_tools build_dumpling + - script: | + # git checkout the enterprise-plugin code. + {{- if (.Git.ref | regexp.Match `^feature/release-[0-9]+\.[0-9]+`) }} + {{ $sv := semver.Semver .Release.version -}} + plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}" + {{- else if (.Git.ref | default "master" | regexp.Match `^(master|main|feature/.*)`) }} + plugin_branch="master" + {{- else }} + {{ $sv := semver.Semver .Release.version -}} + plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}" + {{- end }} + git clone https://github.com/pingcap-inc/enterprise-plugin.git -b "$plugin_branch" ../enterprise-plugin + - script: | + # go plugin audit + pushd ../enterprise-plugin/audit && go mod tidy && popd + go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/audit -out-dir bin/ + - script: | + # go plugin whitelist + pushd ../enterprise-plugin/whitelist && go mod tidy && popd + go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/whitelist -out-dir bin/ debug: - script: | make failpoint-enable WITH_RACE=1 make server build_tools build_dumpling # failpoint make failpoint-disable fips: - - script: ENABLE_FIPS=1 make server build_tools build_dumpling + - script: ENABLE_FIPS=1 make server build_tools build_dumpling artifacts: + - name: "plugins-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + if: {{ eq "enterprise" .Release.profile }} + files: + - name: whitelist-1.so + src: + path: bin/whitelist-1.so + - name: audit-1.so + src: + path: bin/audit-1.so - name: "tidb-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" files: - name: tidb-server @@ -469,11 +501,168 @@ components: type: image artifactory: repo: hub.pingcap.net/pingcap/tidb/images/tidb-server + {{- if eq .Release.profile "enterprise" }} + dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb-enterprise.Dockerfile + {{- else }} dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb.Dockerfile + {{- end }} files: # prepare for context - name: tidb-server src: path: bin/tidb-server + - name: audit-1.so + if: {{ eq "enterprise" .Release.profile }} + src: + path: bin/audit-1.so + - name: whitelist-1.so + if: {{ eq "enterprise" .Release.profile }} + src: + path: bin/whitelist-1.so + - name: container image - br tool + type: image + artifactory: + repo: hub.pingcap.net/pingcap/tidb/images/br + dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/br.Dockerfile + files: # prepare for context + - name: br + src: + path: bin/br + - name: container image - dumpling tool + type: image + artifactory: + repo: hub.pingcap.net/pingcap/tidb/images/dumpling + dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/dumpling.Dockerfile + files: # prepare for context + - name: dumpling + src: + path: bin/dumpling + - name: container image - tidb-lightning + type: image + artifactory: + repo: hub.pingcap.net/pingcap/tidb/images/tidb-lightning + dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb-lightning.Dockerfile + files: # prepare for context + - name: tidb-lightning + src: + path: bin/tidb-lightning + - name: tidb-lightning-ctl + src: + path: bin/tidb-lightning-ctl + - description: From 6.1.0 until 7.0.x + # ref: https://github.com/Masterminds/semver#checking-version-constraints + # fips profile only invalid in v6.5.x + if: {{ or (semver.CheckConstraint ">= 6.1.0-0, < 7.1.0-0" .Release.version) (and (semver.CheckConstraint "~6.5.0-0" .Release.version)) (eq "fips" .Release.profile) }} + os: [linux, darwin] + arch: [amd64, arm64] + profile: [release, debug, fips, enterprise] + steps: + release: + - script: | + make server build_tools build_dumpling + enterprise: + - script: | + TIDB_EDITION=Enterprise make server build_tools build_dumpling + - script: | + # git checkout the enterprise-plugin code. + {{- if (.Git.ref | regexp.Match `^feature/release-[0-9]+\.[0-9]+`) }} + {{ $sv := semver.Semver .Release.version -}} + plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}" + {{- else if (.Git.ref | default "master" | regexp.Match `^(master|main|feature/.*)`) }} + plugin_branch="master" + {{- else }} + {{ $sv := semver.Semver .Release.version -}} + plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}" + {{- end }} + git clone https://github.com/pingcap-inc/enterprise-plugin.git -b "$plugin_branch" ../enterprise-plugin + - script: | + # go plugin audit + pushd ../enterprise-plugin/audit && go mod tidy && popd + go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/audit -out-dir bin/ + - script: | + # go plugin whitelist + pushd ../enterprise-plugin/whitelist && go mod tidy && popd + go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/whitelist -out-dir bin/ + debug: + - script: | + make failpoint-enable + WITH_RACE=1 make server build_tools build_dumpling # failpoint + make failpoint-disable + fips: + - script: ENABLE_FIPS=1 make server build_tools build_dumpling + artifacts: + - name: "plugins-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + if: {{ eq "enterprise" .Release.profile }} + files: + - name: whitelist-1.so + src: + path: bin/whitelist-1.so + - name: audit-1.so + src: + path: bin/audit-1.so + - name: "tidb-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + files: + - name: tidb-server + src: + path: bin/tidb-server + tiup: + description: >- + TiDB is an open source distributed HTAP database compatible with the MySQL protocol. + entrypoint: tidb-server + - name: "br-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + files: + - name: br + src: + path: bin/br + tiup: + description: >- + TiDB/TiKV cluster backup restore tool. + standalone: true + entrypoint: br + - name: "dumpling-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + files: + - name: dumpling + src: + path: bin/dumpling + tiup: + entrypoint: dumpling + description: >- + Dumpling is a CLI tool that helps you dump MySQL/TiDB data. + - name: "tidb-lightning-ctl-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + files: + - name: tidb-lightning-ctl + src: + path: bin/tidb-lightning-ctl + - name: "tidb-lightning-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz" + files: + - name: tidb-lightning + src: + path: bin/tidb-lightning + tiup: + description: >- + TiDB Lightning is a tool used for fast full import of large amounts of data into a TiDB cluster + entrypoint: tidb-lightning + standalone: true + - name: container image - tidb server + type: image + artifactory: + repo: hub.pingcap.net/pingcap/tidb/images/tidb-server + {{- if eq .Release.profile "enterprise" }} + dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb-enterprise.Dockerfile + {{- else }} + dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb.Dockerfile + {{- end }} + files: # prepare for context + - name: tidb-server + src: + path: bin/tidb-server + - name: audit-1.so + if: {{ eq "enterprise" .Release.profile }} + src: + path: bin/audit-1.so + - name: whitelist-1.so + if: {{ eq "enterprise" .Release.profile }} + src: + path: bin/whitelist-1.so - name: container image - br tool type: image artifactory: diff --git a/packages/scripts/build-package-artifacts.sh.tmpl b/packages/scripts/build-package-artifacts.sh.tmpl index 3d61e0de..f7492a01 100644 --- a/packages/scripts/build-package-artifacts.sh.tmpl +++ b/packages/scripts/build-package-artifacts.sh.tmpl @@ -87,7 +87,7 @@ function archive() { release_ws=$(realpath $release_ws) ################# create tarballs ################ - {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }} + {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }} # >>>>>>>>>>>>>>>> file: {{ .name }} >>>>>>>>>>>>>>>> echo "Start composing artifact: {{ .name }} ..." local archive_dir="$release_ws/$(uuidgen)" @@ -102,7 +102,7 @@ function archive() { ## local files {{- range $localFiles }} # - {{ .name }} - mkdir -p $(dirname ${archive_dir}/{{ .name }}) + mkdir -p "$(dirname ${archive_dir}/{{ .name }})" {{- if has .src "extract" }} tar -zxvf {{ .src.path }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path) | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }} mv {{ default "" .src.extract_inner_path }} ${archive_dir}/{{ .name }} @@ -165,12 +165,14 @@ function prepare_artifact_config() { yq -p json -o json -i '.version = "{{ .version }}"' "$save_file" yq -p json -o json -i '.tiup = []' "$save_file" - {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .tiup != null))`) }} + {{- if eq .profile "release" }} + {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false and .tiup != null))`) }} {{- with .tiup }} yq -p json -o json -i '.tiup += {{ . | toJSON }}' "$save_file" {{- end }} yq -p json -o json -i '.tiup[-1].file = "{{ .name }}"' "$save_file" {{- end }} + {{- end }} } function push_files() { @@ -188,8 +190,8 @@ function push_files() { pushd "$release_ws" oras push --artifact-type application/gzip --config artifact-config.json ${destination} - {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }}{{ printf " %s" .name }}{{- end }} - {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }}{{ printf " %s.sha256" .name }}{{- end }} + {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }}{{ printf " %s" .name }}{{- end }} + {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }}{{ printf " %s.sha256" .name }}{{- end }} popd {{- $tag_suffix := printf "-%s_%s_%s" .profile .os .arch -}} @@ -217,7 +219,7 @@ oci: tag: "$tag" digest: "$digest" files: - {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }} + {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }} {{ printf "- %s" .name }} {{ printf "- %s.sha256" .name }} {{- end }} diff --git a/packages/scripts/build-package-images.sh.tmpl b/packages/scripts/build-package-images.sh.tmpl index d5d65ef9..fda57d7f 100644 --- a/packages/scripts/build-package-images.sh.tmpl +++ b/packages/scripts/build-package-images.sh.tmpl @@ -80,7 +80,7 @@ build_and_push_images() { ################# build and push image ################ tag="{{ index .artifactory.tags 0 }}-{{ .profile }}_{{ .os }}_{{ .arch }}" -{{ range (.artifacts | jq `map(select(.type == "image"))`) }} +{{ range (.artifacts | jq `map(select(.type == "image" and .if != false))`) }} # >>>>>>>>>>>>>>>> image: {{ .name }} >>>>>>>>>>>>>>>> archive_dir="$release_ws/tmp-$(date +%s)" mkdir -p "$archive_dir" @@ -182,7 +182,7 @@ write_push_results() { cat < "$result_file" images: -{{- range (.artifacts | jq `map(select(.type == "image"))`) }} +{{- range (.artifacts | jq `map(select(.type == "image" and .if != false))`) }} - repo: {{ .artifactory.repo }} url: "{{ .artifactory.repo }}:$tag" tag: "$tag" From 665f2ee664c027d7e3599de560443a3d61f948ea Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Tue, 26 Dec 2023 15:54:10 +0800 Subject: [PATCH 2/2] ci(packages): add ci steps to verify enterprise profile Signed-off-by: wuhuizuo --- packages/scripts/ci.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/scripts/ci.sh b/packages/scripts/ci.sh index 2fbd9c41..038990b6 100755 --- a/packages/scripts/ci.sh +++ b/packages/scripts/ci.sh @@ -20,6 +20,17 @@ function test_get_builder() { done done + # tidb enterprise profile + local cm="tidb" + for version in $versions; do + for os in $operating_systems; do + for ac in $architectures; do + echo "$cm $os $ac $version:" + $script $cm $os $ac $version $profile + done + done + done + ##### others that owns theirs non-unified versions ##### # tidb-operator local cm="tidb-operator" @@ -71,6 +82,17 @@ function test_gen_package_artifacts_script() { done done + # tidb enterprise profile + local cm="tidb" + for version in $versions; do + for os in $operating_systems; do + for ac in $architectures; do + echo "$cm $os $ac $version:" + $script $cm $os $ac $version $profile branch-xxx 123456789abcdef + done + done + done + ##### others that owns theirs non-unified versions ##### # tidb-operator local cm="tidb-operator" @@ -119,6 +141,15 @@ function test_gen_package_images_script() { done done + # tidb enterprise profile + local cm="tidb" + for version in $versions; do + for ac in $architectures; do + echo "$cm $os $ac $version:" + $script $cm linux $ac $version $profile branch-xxx 123456789abcdef + done + done + ##### others that owns theirs non-unified versions ##### # tidb-operator local cm="tidb-operator"