feat(packages): support enterprise build profile for tidb

packages/README.md

@@ -10,6 +10,13 @@ Central declarative congfigurations for artifacts delivering.
 - [yq]
 - jq
 
+## Profiles
+
+- `release`: community release profile.
+- `debug`: test or debug profile.
+- `enterprise`: enterprise release profile, it will not publish any tiup pkgs.
+- `fips`: fips feature release without enterprise plugins.
+
 ## For component binaries packages and container images
 
 Configuration template: [packages.yaml.tmpl](./packages.yaml.tmpl)

packages/packages.yaml.tmpl

@@ -403,25 +403,57 @@ components:
       - if: {{ semver.CheckConstraint "< 6.1.0-0" .Release.version }}
         image: ghcr.io/pingcap-qe/cd/builders/tidb:v20231115-e1c4b43-go1.18
     routers:
-      - description: From 6.1.0
+      - description: From 7.1.0
         # ref: https://github.com/Masterminds/semver#checking-version-constraints
         # fips profile only invalid in v6.5.x
-        if: {{ or (semver.CheckConstraint ">= 6.1.0-0" .Release.version) (and (semver.CheckConstraint "~6.5.0-0" .Release.version)) (eq "fips" .Release.profile) }}
+        if: {{ semver.CheckConstraint ">= 7.1.0-0" .Release.version }}
         os: [linux, darwin]
         arch: [amd64, arm64]
-        profile: [release, debug, fips]
+        profile: [release, debug, enterprise]
         steps:
           release:
             - script: |
                 make server build_tools build_dumpling
+          enterprise:
+            - script: |
+                TIDB_EDITION=Enterprise make enterprise-prepare enterprise-server-build build_tools build_dumpling
+            - script: |
+                # git checkout the enterprise-plugin code.
+                {{- if (.Git.ref | regexp.Match `^feature/release-[0-9]+\.[0-9]+`) }}
+                {{ $sv := semver.Semver .Release.version -}}
+                plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}"
+                {{- else if (.Git.ref | default "master" | regexp.Match `^(master|main|feature/.*)`) }}
+                plugin_branch="master"
+                {{- else }}
+                {{ $sv := semver.Semver .Release.version -}}
+                plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}"
+                {{- end }}
+                git clone https://github.com/pingcap-inc/enterprise-plugin.git -b "$plugin_branch" ../enterprise-plugin
+            - script: |
+                # go plugin audit
+                pushd ../enterprise-plugin/audit && go mod tidy &&  popd
+                go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/audit -out-dir bin/
+            - script: |
+                # go plugin whitelist
+                pushd ../enterprise-plugin/whitelist && go mod tidy &&  popd
+                go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/whitelist -out-dir bin/
           debug:
             - script: |
                 make failpoint-enable
                 WITH_RACE=1 make server build_tools build_dumpling # failpoint
                 make failpoint-disable
           fips:
-            - script: ENABLE_FIPS=1 make server build_tools build_dumpling                
+            - script: ENABLE_FIPS=1 make server build_tools build_dumpling
         artifacts:
+          - name: "plugins-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            if: {{ eq "enterprise" .Release.profile }}
+            files:
+              - name: whitelist-1.so
+                src:
+                  path: bin/whitelist-1.so
+              - name: audit-1.so
+                src:
+                  path: bin/audit-1.so
           - name: "tidb-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
             files:
               - name: tidb-server
@@ -469,11 +501,168 @@ components:
             type: image
             artifactory:
               repo: hub.pingcap.net/pingcap/tidb/images/tidb-server
+            {{- if eq .Release.profile "enterprise" }}
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb-enterprise.Dockerfile
+            {{- else }}
             dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb.Dockerfile
+            {{- end }}
             files: # prepare for context
               - name: tidb-server
                 src:
                   path: bin/tidb-server
+              - name: audit-1.so
+                if: {{ eq "enterprise" .Release.profile }}
+                src:
+                  path: bin/audit-1.so
+              - name: whitelist-1.so
+                if: {{ eq "enterprise" .Release.profile }}
+                src:
+                  path: bin/whitelist-1.so
+          - name: container image - br tool
+            type: image
+            artifactory:
+              repo: hub.pingcap.net/pingcap/tidb/images/br
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/br.Dockerfile
+            files: # prepare for context
+              - name: br
+                src:
+                  path: bin/br
+          - name: container image - dumpling tool
+            type: image
+            artifactory:
+              repo: hub.pingcap.net/pingcap/tidb/images/dumpling
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/dumpling.Dockerfile
+            files: # prepare for context
+              - name: dumpling
+                src:
+                  path: bin/dumpling
+          - name: container image - tidb-lightning
+            type: image
+            artifactory:
+              repo: hub.pingcap.net/pingcap/tidb/images/tidb-lightning
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb-lightning.Dockerfile
+            files: # prepare for context
+              - name: tidb-lightning
+                src:
+                  path: bin/tidb-lightning
+              - name: tidb-lightning-ctl
+                src:
+                  path: bin/tidb-lightning-ctl
+      - description: From 6.1.0 until 7.0.x
+        # ref: https://github.com/Masterminds/semver#checking-version-constraints
+        # fips profile only invalid in v6.5.x
+        if: {{ or (semver.CheckConstraint ">= 6.1.0-0, < 7.1.0-0" .Release.version) (and (semver.CheckConstraint "~6.5.0-0" .Release.version)) (eq "fips" .Release.profile) }}
+        os: [linux, darwin]
+        arch: [amd64, arm64]
+        profile: [release, debug, fips, enterprise]
+        steps:
+          release:
+            - script: |
+                make server build_tools build_dumpling
+          enterprise:
+            - script: |
+                TIDB_EDITION=Enterprise make server build_tools build_dumpling
+            - script: |
+                # git checkout the enterprise-plugin code.
+                {{- if (.Git.ref | regexp.Match `^feature/release-[0-9]+\.[0-9]+`) }}
+                {{ $sv := semver.Semver .Release.version -}}
+                plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}"
+                {{- else if (.Git.ref | default "master" | regexp.Match `^(master|main|feature/.*)`) }}
+                plugin_branch="master"
+                {{- else }}
+                {{ $sv := semver.Semver .Release.version -}}
+                plugin_branch="{{ printf "release-%d.%d" $sv.Major $sv.Minor }}"
+                {{- end }}
+                git clone https://github.com/pingcap-inc/enterprise-plugin.git -b "$plugin_branch" ../enterprise-plugin
+            - script: |
+                # go plugin audit
+                pushd ../enterprise-plugin/audit && go mod tidy &&  popd
+                go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/audit -out-dir bin/
+            - script: |
+                # go plugin whitelist
+                pushd ../enterprise-plugin/whitelist && go mod tidy &&  popd
+                go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/whitelist -out-dir bin/
+          debug:
+            - script: |
+                make failpoint-enable
+                WITH_RACE=1 make server build_tools build_dumpling # failpoint
+                make failpoint-disable
+          fips:
+            - script: ENABLE_FIPS=1 make server build_tools build_dumpling
+        artifacts:
+          - name: "plugins-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            if: {{ eq "enterprise" .Release.profile }}
+            files:
+              - name: whitelist-1.so
+                src:
+                  path: bin/whitelist-1.so
+              - name: audit-1.so
+                src:
+                  path: bin/audit-1.so
+          - name: "tidb-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            files:
+              - name: tidb-server
+                src:
+                  path: bin/tidb-server
+            tiup:
+              description: >-
+                TiDB is an open source distributed HTAP database compatible with the MySQL protocol.
+              entrypoint: tidb-server
+          - name: "br-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            files:
+              - name: br
+                src:
+                  path: bin/br
+            tiup:
+              description: >-
+                TiDB/TiKV cluster backup restore tool.
+              standalone: true
+              entrypoint: br
+          - name: "dumpling-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            files:
+              - name: dumpling
+                src:
+                  path: bin/dumpling
+            tiup:
+              entrypoint: dumpling
+              description: >-
+                Dumpling is a CLI tool that helps you dump MySQL/TiDB data.
+          - name: "tidb-lightning-ctl-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            files:
+              - name: tidb-lightning-ctl
+                src:
+                  path: bin/tidb-lightning-ctl
+          - name: "tidb-lightning-{{ .Release.version }}-{{ .Release.os }}-{{ .Release.arch }}.tar.gz"
+            files:
+              - name: tidb-lightning
+                src:
+                  path: bin/tidb-lightning
+            tiup:
+              description: >-
+                TiDB Lightning is a tool used for fast full import of large amounts of data into a TiDB cluster
+              entrypoint: tidb-lightning
+              standalone: true
+          - name: container image - tidb server
+            type: image
+            artifactory:
+              repo: hub.pingcap.net/pingcap/tidb/images/tidb-server
+            {{- if eq .Release.profile "enterprise" }}
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb-enterprise.Dockerfile
+            {{- else }}
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/products/tidb.Dockerfile
+            {{- end }}
+            files: # prepare for context
+              - name: tidb-server
+                src:
+                  path: bin/tidb-server
+              - name: audit-1.so
+                if: {{ eq "enterprise" .Release.profile }}
+                src:
+                  path: bin/audit-1.so
+              - name: whitelist-1.so
+                if: {{ eq "enterprise" .Release.profile }}
+                src:
+                  path: bin/whitelist-1.so
           - name: container image - br tool
             type: image
             artifactory:

packages/scripts/build-package-artifacts.sh.tmpl

@@ -87,7 +87,7 @@ function archive() {
     release_ws=$(realpath $release_ws)
 
     ################# create tarballs ################
-    {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }}
+    {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }}
     # >>>>>>>>>>>>>>>> file: {{ .name }} >>>>>>>>>>>>>>>>
     echo "Start composing artifact: {{ .name }} ..."
     local archive_dir="$release_ws/$(uuidgen)"
@@ -102,7 +102,7 @@ function archive() {
     ## local files
     {{- range $localFiles }}
     # - {{ .name }}
-    mkdir -p $(dirname ${archive_dir}/{{ .name }})
+    mkdir -p "$(dirname ${archive_dir}/{{ .name }})"
     {{- if has .src "extract" }}
     tar -zxvf {{ .src.path }} --strip-components={{ math.Sub (strings.Split `/` (default "" .src.extract_inner_path) | len) 1 }} -C $archive_dir {{ default "" .src.extract_inner_path }}
     mv {{ default "" .src.extract_inner_path }} ${archive_dir}/{{ .name }}
@@ -165,12 +165,14 @@ function prepare_artifact_config() {
     yq -p json -o json -i '.version = "{{ .version }}"' "$save_file"
     yq -p json -o json -i '.tiup = []' "$save_file"
 
-    {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .tiup != null))`) }}
+    {{- if eq .profile "release" }}
+    {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false and .tiup != null))`) }}
     {{- with .tiup }}
     yq -p json -o json -i '.tiup += {{ . | toJSON }}' "$save_file"
     {{- end }}
     yq -p json -o json -i '.tiup[-1].file = "{{ .name }}"' "$save_file"
     {{- end }}
+    {{- end }}
 }
 
 function push_files() {
@@ -188,8 +190,8 @@ function push_files() {
 
     pushd "$release_ws"
         oras push --artifact-type application/gzip --config artifact-config.json ${destination}
-        {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }}{{ printf " %s" .name }}{{- end }}
-        {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }}{{ printf " %s.sha256" .name }}{{- end }}
+        {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }}{{ printf " %s" .name }}{{- end }}
+        {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }}{{ printf " %s.sha256" .name }}{{- end }}
     popd
 
     {{- $tag_suffix := printf "-%s_%s_%s" .profile .os .arch -}}
@@ -217,7 +219,7 @@ oci:
   tag: "$tag"
   digest: "$digest"
 files:
-  {{- range (.artifacts | jq `map(select(.type == "file" or .type == null))`) }}
+  {{- range (.artifacts | jq `map(select((.type == "file" or .type == null) and .if != false))`) }}
   {{ printf "- %s" .name }}
   {{ printf "- %s.sha256" .name }}
   {{- end }}

packages/scripts/build-package-images.sh.tmpl

@@ -80,7 +80,7 @@ build_and_push_images() {
 
     ################# build and push image ################
     tag="{{ index .artifactory.tags 0 }}-{{ .profile }}_{{ .os }}_{{ .arch }}"
-{{ range (.artifacts | jq `map(select(.type == "image"))`) }}
+{{ range (.artifacts | jq `map(select(.type == "image" and .if != false))`) }}
     # >>>>>>>>>>>>>>>> image: {{ .name }} >>>>>>>>>>>>>>>>
     archive_dir="$release_ws/tmp-$(date +%s)"
     mkdir -p "$archive_dir"
@@ -182,7 +182,7 @@ write_push_results() {
 
     cat <<EOF > "$result_file"
 images:
-{{- range (.artifacts | jq `map(select(.type == "image"))`) }}
+{{- range (.artifacts | jq `map(select(.type == "image" and .if != false))`) }}
 - repo: {{ .artifactory.repo  }}
   url: "{{ .artifactory.repo  }}:$tag"
   tag: "$tag"

packages/scripts/ci.sh

@@ -20,6 +20,17 @@ function test_get_builder() {
         done
     done
 
+    # tidb enterprise profile
+    local cm="tidb"
+    for version in $versions; do
+        for os in $operating_systems; do
+            for ac in $architectures; do
+                echo "$cm $os $ac $version:"
+                $script $cm $os $ac $version $profile
+            done
+        done
+    done
+
     ##### others that owns theirs non-unified versions #####
     # tidb-operator
     local cm="tidb-operator"
@@ -71,6 +82,17 @@ function test_gen_package_artifacts_script() {
         done
     done
 
+    # tidb enterprise profile
+    local cm="tidb"
+    for version in $versions; do
+        for os in $operating_systems; do
+            for ac in $architectures; do
+                echo "$cm $os $ac $version:"
+                $script $cm $os $ac $version $profile branch-xxx 123456789abcdef
+            done
+        done
+    done
+
     ##### others that owns theirs non-unified versions #####
     # tidb-operator
     local cm="tidb-operator"
@@ -119,6 +141,15 @@ function test_gen_package_images_script() {
         done
     done
 
+    # tidb enterprise profile
+    local cm="tidb"
+    for version in $versions; do
+        for ac in $architectures; do
+            echo "$cm $os $ac $version:"
+            $script $cm linux $ac $version $profile branch-xxx 123456789abcdef
+        done
+    done
+
     ##### others that owns theirs non-unified versions #####
     # tidb-operator
     local cm="tidb-operator"