Merge branch 'master' into ray-project#47991-C416

kenchung285 · Jan 24, 2025 · dc3a2ae · dc3a2ae
2 parents d361122 + 1f83bc9
commit dc3a2ae
Show file tree

Hide file tree

Showing 157 changed files with 2,483 additions and 4,935 deletions.
diff --git a/.buildkite/_forge.rayci.yml b/.buildkite/_forge.rayci.yml
@@ -24,6 +24,7 @@ steps:
           - "11.8.0-cudnn8"
           - "12.1.1-cudnn8"
           - "12.3.2-cudnn9"
+          - "12.4.1-cudnn"
     env:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"

diff --git a/.buildkite/build.rayci.yml b/.buildkite/build.rayci.yml
@@ -59,7 +59,7 @@ steps:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
         --platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
         --platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
-        --platform cpu
+        --platform cu12.4.1-cudnn --platform cpu
         --image-type ray --upload
     depends_on:
       - manylinux

diff --git a/.buildkite/linux_aarch64.rayci.yml b/.buildkite/linux_aarch64.rayci.yml
@@ -32,6 +32,7 @@ steps:
           - "11.8.0-cudnn8"
           - "12.1.1-cudnn8"
           - "12.3.2-cudnn9"
+          - "12.4.1-cudnn"
     instance_type: builder-arm64
     env:
       PYTHON_VERSION: "{{matrix.python}}"
@@ -81,7 +82,7 @@ steps:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
         --platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
         --platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
-        --platform cpu
+        --platform cu12.4.1-cudnn --platform cpu
         --image-type ray --architecture aarch64 --upload
     depends_on:
       - manylinux-aarch64

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -135,7 +135,7 @@ repos:
 
   - repo: local
     hooks:
-      - id: check_cpp_files_inclusion
+      - id: check-cpp-files-inclusion
         name: Check ray core C++ files inclusion violations
         entry: ci/lint/check_cpp_files_inclusion.py
         language: python

diff --git a/ci/docker/base.gpu.Dockerfile b/ci/docker/base.gpu.Dockerfile
@@ -18,12 +18,6 @@ ENV RAY_INSTALL_JAVA=0
 ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
 ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
 ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
-# For wheel build
-# https://github.com/docker-library/docker/blob/master/20.10/docker-entrypoint.sh
-ENV DOCKER_TLS_CERTDIR=/certs
-ENV DOCKER_HOST=tcp://docker:2376
-ENV DOCKER_TLS_VERIFY=1
-ENV DOCKER_CERT_PATH=/certs/client
 ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
 ENV BUILDKITE_BAZEL_CACHE_URL=${REMOTE_CACHE_URL}
 

diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
@@ -14,12 +14,6 @@ ENV PYTHON=$PYTHON
 ENV RAY_USE_RANDOM_PORTS=1
 ENV RAY_DEFAULT_BUILD=1
 ENV RAY_INSTALL_JAVA=0
-# For wheel build
-# https://github.com/docker-library/docker/blob/master/20.10/docker-entrypoint.sh
-ENV DOCKER_TLS_CERTDIR=/certs
-ENV DOCKER_HOST=tcp://docker:2376
-ENV DOCKER_TLS_VERIFY=1
-ENV DOCKER_CERT_PATH=/certs/client
 ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
 
 RUN <<EOF

diff --git a/ci/docker/core.build.Dockerfile b/ci/docker/core.build.Dockerfile
@@ -3,12 +3,6 @@ FROM $DOCKER_IMAGE_BASE_BUILD
 
 ARG RAYCI_IS_GPU_BUILD=false
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/data-tfxbsl.build.Dockerfile b/ci/docker/data-tfxbsl.build.Dockerfile
@@ -7,12 +7,6 @@ ARG ARROW_VERSION=14.*
 ARG ARROW_MONGO_VERSION=
 ARG RAY_CI_JAVA_BUILD=
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/data.build.Dockerfile b/ci/docker/data.build.Dockerfile
@@ -7,12 +7,6 @@ ARG ARROW_VERSION=14.*
 ARG ARROW_MONGO_VERSION=
 ARG RAY_CI_JAVA_BUILD=
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/doc.build.Dockerfile b/ci/docker/doc.build.Dockerfile
@@ -1,12 +1,6 @@
 ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build
 FROM $DOCKER_IMAGE_BASE_BUILD
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/docgpu.build.Dockerfile b/ci/docker/docgpu.build.Dockerfile
@@ -3,12 +3,6 @@
 ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu
 FROM $DOCKER_IMAGE_BASE_BUILD
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/doctest.build.Dockerfile b/ci/docker/doctest.build.Dockerfile
@@ -1,12 +1,6 @@
 ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build
 FROM $DOCKER_IMAGE_BASE_BUILD
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/min.build.Dockerfile b/ci/docker/min.build.Dockerfile
@@ -6,12 +6,6 @@ FROM $DOCKER_IMAGE_BASE_BUILD
 ARG PYTHON_VERSION
 ARG EXTRA_DEPENDENCY
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/ml.build.Dockerfile b/ci/docker/ml.build.Dockerfile
@@ -7,12 +7,6 @@ ARG RAYCI_IS_GPU_BUILD=false
 ARG RAYCI_LIGHTNING_2=false
 ARG PYTHON
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/rllib.build.Dockerfile b/ci/docker/rllib.build.Dockerfile
@@ -3,12 +3,6 @@ FROM $DOCKER_IMAGE_BASE_BUILD
 
 ARG RAYCI_IS_GPU_BUILD
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/serve.build.Dockerfile b/ci/docker/serve.build.Dockerfile
@@ -6,12 +6,6 @@ FROM $DOCKER_IMAGE_BASE_BUILD
 ARG PYDANTIC_VERSION
 ARG PYTHON
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/docker/serverless.build.Dockerfile b/ci/docker/serverless.build.Dockerfile
@@ -1,12 +1,6 @@
 ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build
 FROM $DOCKER_IMAGE_BASE_BUILD
 
-# Unset dind settings; we are using the host's docker daemon.
-ENV DOCKER_TLS_CERTDIR=
-ENV DOCKER_HOST=
-ENV DOCKER_TLS_VERIFY=
-ENV DOCKER_CERT_PATH=
-
 SHELL ["/bin/bash", "-ice"]
 
 COPY . .

diff --git a/ci/lint/git-clang-format b/ci/lint/git-clang-format
@@ -590,7 +590,6 @@ def apply_changes(old_tree, new_tree, force=False, patch_mode=False):
         # right, since it won't be applied to the user's index, but oh well.
         with temporary_index_file(old_tree):
             subprocess.check_call(["git", "checkout", "--patch", new_tree])
-        index_tree = old_tree
     else:
         with temporary_index_file(new_tree):
             run("git", "checkout-index", "-a", "-f")

diff --git a/ci/lint/lint.sh b/ci/lint/lint.sh
@@ -12,10 +12,29 @@ clang_format() {
 
 pre_commit() {
   # Run pre-commit on all files
-  # TODO(MortalHappiness): Run all pre-commit checks
-  # Currently, we only run the ruff format check
-  pip install -c python/requirements_compiled.txt pre-commit
-  pre-commit run ruff --all-files --show-diff-on-failure
+  # TODO(MortalHappiness): Run all pre-commit checks because currently we only run some of them.
+  pip install -c python/requirements_compiled.txt pre-commit clang-format
+
+  HOOKS=(
+    ruff
+    check-added-large-files
+    check-ast
+    check-toml
+    black
+    prettier
+    mypy
+    rst-directive-colons
+    rst-inline-touching-normal
+    python-check-mock-methods
+    clang-format
+    docstyle
+    check-import-order
+    check-cpp-files-inclusion
+  )
+
+  for HOOK in "${HOOKS[@]}"; do
+    pre-commit run "$HOOK" --all-files --show-diff-on-failure
+  done
 }
 
 code_format() {

diff --git a/ci/ray_ci/docker_container.py b/ci/ray_ci/docker_container.py
@@ -13,6 +13,7 @@
     "cu11.8.0-cudnn8",
     "cu12.1.1-cudnn8",
     "cu12.3.2-cudnn9",
+    "cu12.4.1-cudnn",
 ]
 PLATFORMS_RAY_ML = [
     "cpu",

diff --git a/ci/ray_ci/test_ray_docker_container.py b/ci/ray_ci/test_ray_docker_container.py
@@ -321,6 +321,9 @@ def test_get_platform_tag(self) -> None:
         container = RayDockerContainer(v, "cu12.3.2-cudnn9", "ray")
         assert container.get_platform_tag() == "-cu123"
 
+        container = RayDockerContainer(v, "cu12.4.1-cudnn", "ray")
+        assert container.get_platform_tag() == "-cu124"
+
     def test_should_upload(self) -> None:
         v = DEFAULT_PYTHON_VERSION
         test_cases = [

diff --git a/doc/source/_static/css/custom.css b/doc/source/_static/css/custom.css
@@ -398,6 +398,6 @@ table.autosummary tr > td:first-child > p > a > code > span {
 }
 
 /* Hide the RTD version switcher since we are using PyData theme one */
-#rtd-footer-container {
-  display: none;
+readthedocs-flyout {
+  display: none !important;
 }