From ef77a68f3fc1723688fcca37a36eec2de52cba50 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 14:07:23 -0700 Subject: [PATCH 01/17] Simplify build-manylinux2014 script --- tests/buildkite/build-manylinux2014-x86_64.sh | 33 ------------------- ...2014-aarch64.sh => build-manylinux2014.sh} | 11 +++++-- tests/buildkite/pipeline.yml | 4 +-- 3 files changed, 11 insertions(+), 37 deletions(-) delete mode 100644 tests/buildkite/build-manylinux2014-x86_64.sh rename tests/buildkite/{build-manylinux2014-aarch64.sh => build-manylinux2014.sh} (86%) diff --git a/tests/buildkite/build-manylinux2014-x86_64.sh b/tests/buildkite/build-manylinux2014-x86_64.sh deleted file mode 100644 index b00616315b8d..000000000000 --- a/tests/buildkite/build-manylinux2014-x86_64.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -source tests/buildkite/conftest.sh - -WHEEL_TAG=manylinux2014_x86_64 -command_wrapper="tests/ci_build/ci_build.sh manylinux2014_x86_64" -python_bin="/opt/python/cp310-cp310/bin/python" - -echo "--- Build binary wheel for ${WHEEL_TAG}" -# Patch to add warning about manylinux2014 variant -patch -p0 < tests/buildkite/manylinux2014_warning.patch -$command_wrapper bash -c \ - "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/" -git checkout python-package/xgboost/core.py # discard the patch - -$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl -$command_wrapper ${python_bin} tests/ci_build/rename_whl.py \ - --wheel-path wheelhouse/*.whl \ - --commit-hash ${BUILDKITE_COMMIT} \ - --platform-tag ${WHEEL_TAG} -rm -rf python-package/dist/ -mkdir python-package/dist/ -mv -v wheelhouse/*.whl python-package/dist/ - -echo "--- Upload Python wheel" -buildkite-agent artifact upload python-package/dist/*.whl -if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -then - aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \ - --acl public-read --no-progress -fi diff --git a/tests/buildkite/build-manylinux2014-aarch64.sh b/tests/buildkite/build-manylinux2014.sh similarity index 86% rename from tests/buildkite/build-manylinux2014-aarch64.sh rename to tests/buildkite/build-manylinux2014.sh index 802db3f66aaf..fc2665062192 100644 --- a/tests/buildkite/build-manylinux2014-aarch64.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -2,10 +2,17 @@ set -euo pipefail +if [ $# -ne 1 ]; then + echo "Usage: $0 {x86_64,aarch64}" + exit 1 +fi + +arch=$1 + source tests/buildkite/conftest.sh -WHEEL_TAG=manylinux2014_aarch64 -command_wrapper="tests/ci_build/ci_build.sh manylinux2014_aarch64" +WHEEL_TAG="manylinux2014_${arch}" +command_wrapper="tests/ci_build/ci_build.sh ${WHEEL_TAG}" python_bin="/opt/python/cp310-cp310/bin/python" echo "--- Build binary wheel for ${WHEEL_TAG}" diff --git a/tests/buildkite/pipeline.yml b/tests/buildkite/pipeline.yml index acdb71dba529..ee9637b8bd25 100644 --- a/tests/buildkite/pipeline.yml +++ b/tests/buildkite/pipeline.yml @@ -72,12 +72,12 @@ steps: agents: queue: linux-amd64-cpu - label: ":console: Build manylinux2014_x86_64 wheel" - command: "tests/buildkite/build-manylinux2014-x86_64.sh" + command: "tests/buildkite/build-manylinux2014.sh x86_64" key: build-manylinux2014-x86_64 agents: queue: linux-amd64-cpu - label: ":console: Build manylinux2014_aarch64 wheel" - command: "tests/buildkite/build-manylinux2014-aarch64.sh" + command: "tests/buildkite/build-manylinux2014.sh aarch64" key: build-manylinux2014-aarch64 agents: queue: linux-arm64-cpu From 9212e1cd80a8c1fedf6f57c8fe516b7e513385dc Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 14:26:28 -0700 Subject: [PATCH 02/17] Mark the script as executable --- tests/buildkite/build-manylinux2014.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/buildkite/build-manylinux2014.sh diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh old mode 100644 new mode 100755 From edfcc5e6f47030efa594a4a95f569581a1365815 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 14:45:14 -0700 Subject: [PATCH 03/17] Fix i386 --- tests/ci_build/Dockerfile.i386 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci_build/Dockerfile.i386 b/tests/ci_build/Dockerfile.i386 index d7c133e2aee4..a582a54020bb 100644 --- a/tests/ci_build/Dockerfile.i386 +++ b/tests/ci_build/Dockerfile.i386 @@ -1,7 +1,7 @@ FROM i386/debian:sid ENV DEBIAN_FRONTEND noninteractive -SHELL ["/bin/bash", "-c"] # Use Bash as shell +SHELL ["/bin/bash", "-c"] RUN \ apt-get update && \ From 7eb15fb5a653f03b3855802a96696f625373ef62 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 14:54:57 -0700 Subject: [PATCH 04/17] Try building xgboost-cpu --- tests/buildkite/build-manylinux2014.sh | 28 ++++++++++++++++++++------ tests/buildkite/cpu_only_pypkg.patch | 23 +++++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/buildkite/cpu_only_pypkg.patch diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh index fc2665062192..d62c3d8f3c07 100755 --- a/tests/buildkite/build-manylinux2014.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -20,7 +20,7 @@ echo "--- Build binary wheel for ${WHEEL_TAG}" patch -p0 < tests/buildkite/manylinux2014_warning.patch $command_wrapper bash -c \ "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/" -git checkout python-package/xgboost/core.py # discard the patch +git checkout python-package/pyproject.toml python-package/xgboost/core.py # discard the patch $command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl $command_wrapper ${python_bin} tests/ci_build/rename_whl.py \ @@ -31,10 +31,26 @@ rm -rf python-package/dist/ mkdir python-package/dist/ mv -v wheelhouse/*.whl python-package/dist/ +echo "--- Build binary wheel for ${WHEEL_TAG} (CPU only)" +patch -p0 < tests/buildkite/cpu_only_pypkg.patch +$command_wrapper bash -c \ + "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/" +git checkout python-package/pyproject.toml # discard the patch + +$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl +$command_wrapper ${python_bin} tests/ci_build/rename_whl.py \ + --wheel-path wheelhouse/*.whl \ + --commit-hash ${BUILDKITE_COMMIT} \ + --platform-tag ${WHEEL_TAG} +mv -v wheelhouse/*.whl python-package/dist/ + echo "--- Upload Python wheel" buildkite-agent artifact upload python-package/dist/*.whl -if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -then - aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \ - --acl public-read --no-progress -fi +# if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +# then + for wheel in python-package/dist/*.whl + do + aws s3 cp "${wheel}" s3://xgboost-nightly-builds/${BRANCH_NAME}/ \ + --acl public-read --no-progress + done +# fi diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch new file mode 100644 index 000000000000..1d00aad6bfbe --- /dev/null +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -0,0 +1,23 @@ +diff --git python-package/pyproject.toml python-package/pyproject.toml +index a273d8c13..fcd93c9eb 100644 +--- python-package/pyproject.toml ++++ python-package/pyproject.toml +@@ -6,7 +6,7 @@ backend-path = ["."] + build-backend = "packager.pep517" + + [project] +-name = "xgboost" ++name = "xgboost-cpu" + version = "2.2.0-dev" + authors = [ + { name = "Hyunsu Cho", email = "chohyu01@cs.washington.edu" }, +@@ -30,8 +30,7 @@ classifiers = [ + ] + dependencies = [ + "numpy", +- "scipy", +- "nvidia-nccl-cu12 ; platform_system == 'Linux' and platform_machine != 'aarch64'" ++ "scipy" + ] + + [project.urls] From 3a41d45dc2174ccaff0a5633295e8d5d7d8c2fea Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 15:22:18 -0700 Subject: [PATCH 05/17] Fix patch --- tests/buildkite/cpu_only_pypkg.patch | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch index 1d00aad6bfbe..819563e66fd7 100644 --- a/tests/buildkite/cpu_only_pypkg.patch +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -1,5 +1,5 @@ diff --git python-package/pyproject.toml python-package/pyproject.toml -index a273d8c13..fcd93c9eb 100644 +index a273d8c13..f6a2c6eb0 100644 --- python-package/pyproject.toml +++ python-package/pyproject.toml @@ -6,7 +6,7 @@ backend-path = ["."] @@ -21,3 +21,13 @@ index a273d8c13..fcd93c9eb 100644 ] [project.urls] +@@ -46,6 +45,9 @@ datatable = ["datatable"] + plotting = ["graphviz", "matplotlib"] + pyspark = ["pyspark", "scikit-learn", "cloudpickle"] + ++[tool.hatch.build.targets.wheel] ++packages = ["xgboost/"] ++ + [tool.hatch.build.targets.wheel.hooks.custom] + + [tool.isort] From 09dcdac2d95c033229b46495209b18ae9adcaa37 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 15:41:44 -0700 Subject: [PATCH 06/17] Fix --- tests/buildkite/build-manylinux2014.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh index d62c3d8f3c07..ac35a1acbd88 100755 --- a/tests/buildkite/build-manylinux2014.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -37,12 +37,12 @@ $command_wrapper bash -c \ "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/" git checkout python-package/pyproject.toml # discard the patch -$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl +$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/xgboost_cpu-*.whl $command_wrapper ${python_bin} tests/ci_build/rename_whl.py \ - --wheel-path wheelhouse/*.whl \ + --wheel-path wheelhouse/xgboost_cpu-*.whl \ --commit-hash ${BUILDKITE_COMMIT} \ --platform-tag ${WHEEL_TAG} -mv -v wheelhouse/*.whl python-package/dist/ +mv -v wheelhouse/xgboost_cpu-*.whl python-package/dist/ echo "--- Upload Python wheel" buildkite-agent artifact upload python-package/dist/*.whl From 394c59983efa22bf5487812d76de299d519f020c Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 17 Jul 2024 17:04:24 -0700 Subject: [PATCH 07/17] Fix --- tests/buildkite/build-manylinux2014.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh index ac35a1acbd88..a4d50c5059a8 100755 --- a/tests/buildkite/build-manylinux2014.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -45,7 +45,10 @@ $command_wrapper ${python_bin} tests/ci_build/rename_whl.py \ mv -v wheelhouse/xgboost_cpu-*.whl python-package/dist/ echo "--- Upload Python wheel" -buildkite-agent artifact upload python-package/dist/*.whl +for wheel in python-package/dist/*.whl +do + buildkite-agent artifact upload "${wheel}" +done # if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] # then for wheel in python-package/dist/*.whl From 3e66f5045f5714f28086ec2988b8fbca26a84e8b Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 09:53:39 -0700 Subject: [PATCH 08/17] Remove dupe wheel --- tests/buildkite/build-manylinux2014.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh index a4d50c5059a8..fdc34eacaa97 100755 --- a/tests/buildkite/build-manylinux2014.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -19,7 +19,7 @@ echo "--- Build binary wheel for ${WHEEL_TAG}" # Patch to add warning about manylinux2014 variant patch -p0 < tests/buildkite/manylinux2014_warning.patch $command_wrapper bash -c \ - "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/" + "cd python-package && ${python_bin} -m pip wheel --no-deps -v . --wheel-dir dist/" git checkout python-package/pyproject.toml python-package/xgboost/core.py # discard the patch $command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl @@ -34,7 +34,7 @@ mv -v wheelhouse/*.whl python-package/dist/ echo "--- Build binary wheel for ${WHEEL_TAG} (CPU only)" patch -p0 < tests/buildkite/cpu_only_pypkg.patch $command_wrapper bash -c \ - "cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/" + "cd python-package && ${python_bin} -m pip wheel --no-deps -v . --wheel-dir dist/" git checkout python-package/pyproject.toml # discard the patch $command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/xgboost_cpu-*.whl @@ -42,6 +42,7 @@ $command_wrapper ${python_bin} tests/ci_build/rename_whl.py \ --wheel-path wheelhouse/xgboost_cpu-*.whl \ --commit-hash ${BUILDKITE_COMMIT} \ --platform-tag ${WHEEL_TAG} +rm -v python-package/dist/xgboost_cpu-*.whl mv -v wheelhouse/xgboost_cpu-*.whl python-package/dist/ echo "--- Upload Python wheel" From 7876a45231a10198055b9d75b6498d46e5ec4beb Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 12:18:44 -0700 Subject: [PATCH 09/17] Re-order fields in pyproject.toml, to minimize diff --- python-package/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index a273d8c135d8..8835def25858 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -7,13 +7,13 @@ build-backend = "packager.pep517" [project] name = "xgboost" -version = "2.2.0-dev" +description = "XGBoost Python Package" +readme = { file = "README.rst", content-type = "text/x-rst" } authors = [ { name = "Hyunsu Cho", email = "chohyu01@cs.washington.edu" }, { name = "Jiaming Yuan", email = "jm.yuan@outlook.com" } ] -description = "XGBoost Python Package" -readme = { file = "README.rst", content-type = "text/x-rst" } +version = "2.2.0-dev" requires-python = ">=3.8" license = { text = "Apache-2.0" } classifiers = [ From f5ec1fbf715421f14f7614f3942a79174eff8269 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 12:20:59 -0700 Subject: [PATCH 10/17] Refine patch --- tests/buildkite/cpu_only_pypkg.patch | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch index 819563e66fd7..1a4e69c3bd45 100644 --- a/tests/buildkite/cpu_only_pypkg.patch +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -1,5 +1,5 @@ diff --git python-package/pyproject.toml python-package/pyproject.toml -index a273d8c13..f6a2c6eb0 100644 +index 8835def25..b9cbebe11 100644 --- python-package/pyproject.toml +++ python-package/pyproject.toml @@ -6,7 +6,7 @@ backend-path = ["."] @@ -8,9 +8,9 @@ index a273d8c13..f6a2c6eb0 100644 [project] -name = "xgboost" +name = "xgboost-cpu" - version = "2.2.0-dev" + description = "XGBoost Python Package" + readme = { file = "README.rst", content-type = "text/x-rst" } authors = [ - { name = "Hyunsu Cho", email = "chohyu01@cs.washington.edu" }, @@ -30,8 +30,7 @@ classifiers = [ ] dependencies = [ @@ -21,13 +21,13 @@ index a273d8c13..f6a2c6eb0 100644 ] [project.urls] -@@ -46,6 +45,9 @@ datatable = ["datatable"] - plotting = ["graphviz", "matplotlib"] - pyspark = ["pyspark", "scikit-learn", "cloudpickle"] +@@ -48,6 +47,9 @@ pyspark = ["pyspark", "scikit-learn", "cloudpickle"] + + [tool.hatch.build.targets.wheel.hooks.custom] +[tool.hatch.build.targets.wheel] +packages = ["xgboost/"] + - [tool.hatch.build.targets.wheel.hooks.custom] - [tool.isort] + profile = "black" + From 8018e7f37e4542c522c5642b20b9559167ccb14c Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 12:25:25 -0700 Subject: [PATCH 11/17] Refine patch --- tests/buildkite/cpu_only_pypkg.patch | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch index 1a4e69c3bd45..d622fea1b4c6 100644 --- a/tests/buildkite/cpu_only_pypkg.patch +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -1,5 +1,5 @@ diff --git python-package/pyproject.toml python-package/pyproject.toml -index 8835def25..b9cbebe11 100644 +index 8835def25..42400a78a 100644 --- python-package/pyproject.toml +++ python-package/pyproject.toml @@ -6,7 +6,7 @@ backend-path = ["."] @@ -21,10 +21,11 @@ index 8835def25..b9cbebe11 100644 ] [project.urls] -@@ -48,6 +47,9 @@ pyspark = ["pyspark", "scikit-learn", "cloudpickle"] +@@ -48,6 +47,10 @@ pyspark = ["pyspark", "scikit-learn", "cloudpickle"] [tool.hatch.build.targets.wheel.hooks.custom] ++ +[tool.hatch.build.targets.wheel] +packages = ["xgboost/"] + From 7b2b25d00b214f522d2c486992989d2821aac870 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 12:28:49 -0700 Subject: [PATCH 12/17] Refine patch --- tests/buildkite/cpu_only_pypkg.patch | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch index d622fea1b4c6..5062faa56882 100644 --- a/tests/buildkite/cpu_only_pypkg.patch +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -1,5 +1,5 @@ diff --git python-package/pyproject.toml python-package/pyproject.toml -index 8835def25..42400a78a 100644 +index 8835def25..c5dc908d9 100644 --- python-package/pyproject.toml +++ python-package/pyproject.toml @@ -6,7 +6,7 @@ backend-path = ["."] @@ -21,14 +21,10 @@ index 8835def25..42400a78a 100644 ] [project.urls] -@@ -48,6 +47,10 @@ pyspark = ["pyspark", "scikit-learn", "cloudpickle"] - - [tool.hatch.build.targets.wheel.hooks.custom] +@@ -83,3 +82,6 @@ class-attribute-naming-style = "snake_case" + # Allow single-letter variables + variable-rgx = "[a-zA-Z_][a-z0-9_]{0,30}$" + +[tool.hatch.build.targets.wheel] +packages = ["xgboost/"] -+ - [tool.isort] - profile = "black" - From 001cdc2f2a07a7ec6d440aebac6bd6b7ca3e0c15 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 16:10:00 -0700 Subject: [PATCH 13/17] Split patch --- tests/buildkite/build-manylinux2014.sh | 3 +++ tests/buildkite/cpu_only_pypkg.patch | 12 +----------- tests/buildkite/manylinux2014_warning.patch | 14 -------------- tests/buildkite/remove_nccl_dep.patch | 14 ++++++++++++++ 4 files changed, 18 insertions(+), 25 deletions(-) create mode 100644 tests/buildkite/remove_nccl_dep.patch diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh index fdc34eacaa97..9227852f87d0 100755 --- a/tests/buildkite/build-manylinux2014.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -17,6 +17,7 @@ python_bin="/opt/python/cp310-cp310/bin/python" echo "--- Build binary wheel for ${WHEEL_TAG}" # Patch to add warning about manylinux2014 variant +patch -p0 < tests/buildkite/remove_nccl_dep.patch patch -p0 < tests/buildkite/manylinux2014_warning.patch $command_wrapper bash -c \ "cd python-package && ${python_bin} -m pip wheel --no-deps -v . --wheel-dir dist/" @@ -32,6 +33,8 @@ mkdir python-package/dist/ mv -v wheelhouse/*.whl python-package/dist/ echo "--- Build binary wheel for ${WHEEL_TAG} (CPU only)" +# Patch to rename pkg to xgboost-cpu +patch -p0 < tests/buildkite/remove_nccl_dep.patch patch -p0 < tests/buildkite/cpu_only_pypkg.patch $command_wrapper bash -c \ "cd python-package && ${python_bin} -m pip wheel --no-deps -v . --wheel-dir dist/" diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch index 5062faa56882..e402a06e3faf 100644 --- a/tests/buildkite/cpu_only_pypkg.patch +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -11,17 +11,7 @@ index 8835def25..c5dc908d9 100644 description = "XGBoost Python Package" readme = { file = "README.rst", content-type = "text/x-rst" } authors = [ -@@ -30,8 +30,7 @@ classifiers = [ - ] - dependencies = [ - "numpy", -- "scipy", -- "nvidia-nccl-cu12 ; platform_system == 'Linux' and platform_machine != 'aarch64'" -+ "scipy" - ] - - [project.urls] -@@ -83,3 +82,6 @@ class-attribute-naming-style = "snake_case" +@@ -82,3 +82,6 @@ class-attribute-naming-style = "snake_case" # Allow single-letter variables variable-rgx = "[a-zA-Z_][a-z0-9_]{0,30}$" diff --git a/tests/buildkite/manylinux2014_warning.patch b/tests/buildkite/manylinux2014_warning.patch index 692a92672d2f..679205988b7a 100644 --- a/tests/buildkite/manylinux2014_warning.patch +++ b/tests/buildkite/manylinux2014_warning.patch @@ -1,17 +1,3 @@ -diff --git python-package/pyproject.toml python-package/pyproject.toml -index a273d8c13..dee49686a 100644 ---- python-package/pyproject.toml -+++ python-package/pyproject.toml -@@ -30,8 +30,7 @@ classifiers = [ - ] - dependencies = [ - "numpy", -- "scipy", -- "nvidia-nccl-cu12 ; platform_system == 'Linux' and platform_machine != 'aarch64'" -+ "scipy" - ] - - [project.urls] diff --git python-package/xgboost/core.py python-package/xgboost/core.py index e8bc735e6..030972ef2 100644 --- python-package/xgboost/core.py diff --git a/tests/buildkite/remove_nccl_dep.patch b/tests/buildkite/remove_nccl_dep.patch new file mode 100644 index 000000000000..a2a4a5c88289 --- /dev/null +++ b/tests/buildkite/remove_nccl_dep.patch @@ -0,0 +1,14 @@ +diff --git python-package/pyproject.toml python-package/pyproject.toml +index 8835def25..46c1451c2 100644 +--- python-package/pyproject.toml ++++ python-package/pyproject.toml +@@ -30,8 +30,7 @@ classifiers = [ + ] + dependencies = [ + "numpy", +- "scipy", +- "nvidia-nccl-cu12 ; platform_system == 'Linux' and platform_machine != 'aarch64'" ++ "scipy" + ] + + [project.urls] From 91c9d2edbcc6629e1015b3d706c29b8f2ac9cbc3 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 17:47:09 -0700 Subject: [PATCH 14/17] Remove NCCL dep from sdist --- dev/release-artifacts.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dev/release-artifacts.py b/dev/release-artifacts.py index 1e0b5723e89b..f53ef134630b 100644 --- a/dev/release-artifacts.py +++ b/dev/release-artifacts.py @@ -2,6 +2,7 @@ tqdm, sh are required to run this script. """ + import argparse import os import shutil @@ -106,6 +107,15 @@ def make_pysrc_wheel( if not os.path.exists(dist): os.mkdir(dist) + # Apply patch to remove NCCL dependency + # Save the original content of pyproject.toml so that we can restore it later + with DirectoryExcursion(ROOT): + with open("python-package/pyproject.toml", "r") as f: + orig_pyproj_lines = f.read() + with open("tests/buildkite/remove_nccl_dep.patch", "r") as f: + patch_lines = f.read() + subprocess.run(["patch", "-p0"], input=patch_lines, text=True) + with DirectoryExcursion(os.path.join(ROOT, "python-package")): subprocess.check_call(["python", "-m", "build", "--sdist"]) if rc is not None: @@ -117,6 +127,10 @@ def make_pysrc_wheel( target = os.path.join(dist, name) shutil.move(src, target) + with DirectoryExcursion(ROOT): + with open("python-package/pyproject.toml", "w") as f: + print(orig_pyproj_lines, file=f, end="") + def download_py_packages( branch: str, major: int, minor: int, commit_hash: str, outdir: str From f8e6a6d59b856d3ea142f4e002a922ba58289858 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 17:57:09 -0700 Subject: [PATCH 15/17] Update README.rst for xgboost-cpu wheel --- tests/buildkite/cpu_only_pypkg.patch | 37 +++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/buildkite/cpu_only_pypkg.patch b/tests/buildkite/cpu_only_pypkg.patch index e402a06e3faf..765ac5c098d0 100644 --- a/tests/buildkite/cpu_only_pypkg.patch +++ b/tests/buildkite/cpu_only_pypkg.patch @@ -1,5 +1,40 @@ +diff --git python-package/README.rst python-package/README.rst +index 1fc0bb5a0..f1c68470b 100644 +--- python-package/README.rst ++++ python-package/README.rst +@@ -1,20 +1,15 @@ +-====================== +-XGBoost Python Package +-====================== ++================================= ++XGBoost Python Package (CPU only) ++================================= + + |PyPI version| + +-Installation +-============ ++The ``xgboost-cpu`` package provides for a minimal installation, with no support for the GPU algorithms ++or federated learning. It is provided to allow XGBoost to be installed in a space-constrained ++environments. + +-From `PyPI `_ +---------------------------------------------------- ++Note. ``xgboost-cpu`` package is only provided for x86_64 (amd64) Linux and Windows platforms. ++For other platforms, please install ``xgboost`` from https://pypi.org/project/xgboost/. + +-For a stable version, install using ``pip``:: +- +- pip install xgboost +- +-.. |PyPI version| image:: https://badge.fury.io/py/xgboost.svg +- :target: http://badge.fury.io/py/xgboost +- +-For building from source, see `build `_. ++Note. ``xgboost-cpu`` does not provide an sdist (source distribution). You may install sdist ++from https://pypi.org/project/xgboost/. diff --git python-package/pyproject.toml python-package/pyproject.toml -index 8835def25..c5dc908d9 100644 +index 46c1451c2..c5dc908d9 100644 --- python-package/pyproject.toml +++ python-package/pyproject.toml @@ -6,7 +6,7 @@ backend-path = ["."] From ac2bdfd097ff0e6ebdc77f514578e2509bf32d9b Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 18:03:36 -0700 Subject: [PATCH 16/17] Update install doc --- doc/install.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/install.rst b/doc/install.rst index e5229702e16c..79082a7ed581 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -76,6 +76,19 @@ Capabilities of binary wheels for each platform: | Windows | |tick| | |cross| | +---------------------+---------+----------------------+ +Minimal installation (CPU-only) +******************************* +The default installation with ``pip`` will install the full XGBoost package, including the support for the GPU algorithms and federated learning. + +You may choose to reduce the size of the installed package and save the disk space, by opting to install ``xgboost-cpu`` instead: + +.. code-block:: bash + + pip install xgboost-cpu + +The ``xgboost-cpu`` variant will have drastically smaller disk footprint, but does not provide some features, such as the GPU algorithms and +federated learning. + Conda ***** From 0c0009b1b8dd3c1e01d91a8aacde9321ace9aee3 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 18 Jul 2024 18:07:39 -0700 Subject: [PATCH 17/17] Remove commented lines --- tests/buildkite/build-manylinux2014.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/buildkite/build-manylinux2014.sh b/tests/buildkite/build-manylinux2014.sh index 9227852f87d0..426d32b5c361 100755 --- a/tests/buildkite/build-manylinux2014.sh +++ b/tests/buildkite/build-manylinux2014.sh @@ -53,11 +53,11 @@ for wheel in python-package/dist/*.whl do buildkite-agent artifact upload "${wheel}" done -# if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -# then +if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +then for wheel in python-package/dist/*.whl do aws s3 cp "${wheel}" s3://xgboost-nightly-builds/${BRANCH_NAME}/ \ --acl public-read --no-progress done -# fi +fi