[CI] Create a central place to update CI image tag

dmlc · Feb 8, 2025 · b583bd5 · b583bd5
1 parent ea66308
commit b583bd5
Show file tree

Hide file tree

Showing 16 changed files with 51 additions and 31 deletions.
diff --git a/doc/contrib/ci.rst b/doc/contrib/ci.rst
@@ -43,18 +43,26 @@ To make changes to the CI container, carry out the following steps:
    Consult :ref:`build_run_docker_locally` for this step.
 4. Submit a pull request to `dmlc/xgboost-devops <https://github.com/dmlc/xgboost-devops>`_ with
    the proposed changes to the Dockerfile. Make note of the pull request number. Example: ``#204``
-5. Clone `dmlc/xgboost <https://github.com/dmlc/xgboost>`_ and update all references to the
-   old container to point to the new container. More specifically, all container image URIs of form
-   ``492475357299.dkr.ecr.us-west-2.amazonaws.com/[image_repo]:main`` should have its image tag
-   (last component) replaced with ``PR-#``, where ``#`` is the pull request number.
-   For the example above,
-   we'd replace ``492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main`` with
-   ``492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:PR-204``.
+5. Clone `dmlc/xgboost <https://github.com/dmlc/xgboost>`_. Locate the file
+   ``ops/pipeline/get-image-tag.sh``, which should have a single line
+
+   .. code-block:: bash
+
+     IMAGE_TAG=main
+
+   To use the new container, revise the file as follows:
+
+   .. code-block:: bash
+
+     IMAGE_TAG=PR-XX
+
+   where ``XX`` is the pull request number.
+
 6. Now submit a pull request to `dmlc/xgboost <https://github.com/dmlc/xgboost>`_. The CI will
    run tests using the new container. Verify that all tests pass.
 7. Merge the pull request in ``dmlc/xgboost-devops``. Wait until the CI completes on the ``main`` branch.
-8. Go back to the the pull request for ``dmlc/xgboost`` and revise all the container references to use
-   the old tag ``:main``.
+8. Go back to the the pull request for ``dmlc/xgboost`` and change ``ops/pipeline/get-image-tag.sh``
+   back to ``IMAGE_TAG=main``.
 9. Merge the pull request in ``dmlc/xgboost``.
 
 .. _build_run_docker_locally:

diff --git a/ops/pipeline/build-cpu-arm64.sh b/ops/pipeline/build-cpu-arm64.sh
@@ -11,9 +11,10 @@ fi
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
 WHEEL_TAG=manylinux_2_28_aarch64
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:${IMAGE_TAG}
 
 echo "--- Build CPU code targeting ARM64"
 set -x

diff --git a/ops/pipeline/build-cpu.sh b/ops/pipeline/build-cpu.sh
@@ -5,8 +5,9 @@ set -euo pipefail
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:${IMAGE_TAG}
 
 echo "--- Build CPU code"
 set -x

diff --git a/ops/pipeline/build-cuda.sh b/ops/pipeline/build-cuda.sh
@@ -33,10 +33,11 @@ esac
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
 WHEEL_TAG=manylinux_2_28_x86_64
-BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main"
-MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:main"
+BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:${IMAGE_TAG}"
+MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:${IMAGE_TAG}"
 
 echo "--- Build with CUDA"
 

diff --git a/ops/pipeline/build-gpu-rpkg.sh b/ops/pipeline/build-gpu-rpkg.sh
@@ -10,8 +10,9 @@ fi
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:${IMAGE_TAG}
 
 echo "--- Build XGBoost R package with CUDA"
 set -x

diff --git a/ops/pipeline/build-jvm-doc.sh b/ops/pipeline/build-jvm-doc.sh
@@ -18,8 +18,9 @@ then
 fi
 
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:${IMAGE_TAG}
 
 echo "--- Build JVM packages doc"
 set -x

diff --git a/ops/pipeline/build-jvm-gpu.sh b/ops/pipeline/build-jvm-gpu.sh
@@ -5,8 +5,9 @@ set -euo pipefail
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:${IMAGE_TAG}
 
 echo "--- Build libxgboost4j.so with CUDA"
 

diff --git a/ops/pipeline/build-jvm-manylinux2014.sh b/ops/pipeline/build-jvm-manylinux2014.sh
@@ -14,8 +14,9 @@ image_repo="xgb-ci.manylinux2014_${arch}"
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main"
+IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:${IMAGE_TAG}"
 
 # Build XGBoost4J binary
 echo "--- Build libxgboost4j.so (targeting glibc 2.17)"

diff --git a/ops/pipeline/build-manylinux2014.sh b/ops/pipeline/build-manylinux2014.sh
@@ -18,10 +18,11 @@ arch="$1"
 
 source ops/pipeline/classify-git-branch.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
 WHEEL_TAG="manylinux2014_${arch}"
 IMAGE_REPO="xgb-ci.${WHEEL_TAG}"
-IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:main"
+IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:${IMAGE_TAG}"
 PYTHON_BIN="/opt/python/cp310-cp310/bin/python"
 
 echo "--- Build binary wheel for ${WHEEL_TAG}"

diff --git a/ops/pipeline/build-test-jvm-packages.sh b/ops/pipeline/build-test-jvm-packages.sh
@@ -13,6 +13,7 @@ EOF
 set -euo pipefail
 
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
 for arg in "SCALA_VERSION"
 do
@@ -23,7 +24,7 @@ do
   fi
 done
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:${IMAGE_TAG}
 
 set -x
 

diff --git a/ops/pipeline/deploy-jvm-packages.sh b/ops/pipeline/deploy-jvm-packages.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 
 source ops/pipeline/enforce-ci.sh
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
 if [[ "$#" -lt 3 ]]
 then
@@ -16,7 +17,7 @@ variant="$1"
 image_repo="$2"
 scala_version="$3"
 
-IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main"
+IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:${IMAGE_TAG}"
 
 set -x
 

diff --git a/ops/pipeline/get-image-tag.sh b/ops/pipeline/get-image-tag.sh
@@ -0,0 +1,4 @@
+## Update the following line to test changes to CI images
+## See https://xgboost.readthedocs.io/en/latest/contrib/ci.html#making-changes-to-ci-containers
+
+IMAGE_TAG=main
diff --git a/ops/pipeline/run-clang-tidy.sh b/ops/pipeline/run-clang-tidy.sh
@@ -3,8 +3,9 @@
 set -euo pipefail
 
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:${IMAGE_TAG}
 
 echo "--- Run clang-tidy"
 set -x

diff --git a/ops/pipeline/test-cpp-i386.sh b/ops/pipeline/test-cpp-i386.sh
@@ -4,8 +4,9 @@
 set -euo pipefail
 
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:main"
+IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:${IMAGE_TAG}"
 
 set -x
 python3 ops/docker_run.py \

diff --git a/ops/pipeline/test-jvm-gpu.sh b/ops/pipeline/test-jvm-gpu.sh
@@ -24,8 +24,9 @@ do
 done
 
 source ops/pipeline/get-docker-registry-details.sh
+source ops/pipeline/get-image-tag.sh
 
-IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main
+IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:${IMAGE_TAG}
 
 set -x
 

diff --git a/ops/pipeline/test-python-wheel.sh b/ops/pipeline/test-python-wheel.sh
@@ -19,15 +19,10 @@ else
   gpu_option=""
 fi
 
-if [[ "$image_repo" == "xgb-ci.gpu" ]]
-then
-  image_tag="PR-11"
-else
-  image_tag="main"
-fi
-
 source ops/pipeline/get-docker-registry-details.sh
-IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:${image_tag}"
+source ops/pipeline/get-image-tag.sh
+
+IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:${IMAGE_TAG}"
 
 set -x
 python3 ops/docker_run.py --image-uri "${IMAGE_URI}" ${gpu_option} \