Merge branch 'branch-23.12' into fea_lrmg_sparse

.github/workflows/build.yaml

@@ -22,7 +22,7 @@ on:
         default: nightly
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
 jobs:

BUILD.md

@@ -19,8 +19,8 @@ It is recommended to use conda for environment/package management. If doing so,
 
 ```bash
 conda create -n cuml_dev python=3.10
+conda env update -n cuml_dev --file=conda/environments/all_cuda-118_arch-x86_64.yaml
 conda activate cuml_dev
-conda env update --file=conda/environments/all_cuda-118_arch-x86_64.yaml
 ```
 
 ## Installing from Source:

VERSION

@@ -0,0 +1 @@
+23.12.00

ci/build_cpp.sh

@@ -9,8 +9,10 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+version=$(rapids-generate-version)
+
 rapids-logger "Begin cpp build"
 
-rapids-conda-retry mambabuild conda/recipes/libcuml
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcuml
 
 rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -9,6 +9,16 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
+package_name="cuml"
+package_dir="python"
+
+version=$(rapids-generate-version)
+git_commit=$(git rev-parse HEAD)
+export RAPIDS_PACKAGE_VERSION=${version} 
+
+echo "${version}" > VERSION
+sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
 rapids-logger "Begin py build"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)

ci/build_wheel.sh

@@ -3,20 +3,51 @@
 
 set -euo pipefail
 
+package_name="cuml"
+package_dir="python"
+
 source rapids-configure-sccache
 source rapids-date-string
 
-# Use gha-tools rapids-pip-wheel-version to generate wheel version then
-# update the necessary files
-version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
+version=$(rapids-generate-version)
+git_commit=$(git rev-parse HEAD)
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}"
-echo "The package name and/or version was modified in the package source. The git diff is:"
-git diff
+# This is the version of the suffix with a preceding hyphen. It's used
+# everywhere except in the final wheel name.
+PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+
+# Patch project metadata files to include the CUDA version suffix and version override.
+pyproject_file="${package_dir}/pyproject.toml"
+
+sed -i "/^name.*cuml/ s/= \"cuml\"/= \"cuml${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
+echo "${version}" > VERSION
+sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/_version.py"
+
+# For nightlies we want to ensure that we're pulling in alphas as well. The
+# easiest way to do so is to augment the spec with a constraint containing a
+# min alpha version that doesn't affect the version bounds but does allow usage
+# of alpha versions for that dependency without --pre
+alpha_spec=''
+if ! rapids-is-release-build; then
+    alpha_spec=',>=0.0.0a0'
+fi
+
+for dep in cudf pylibraft raft-dask rmm; do
+    sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
+done
+
+for dep in dask-cuda rapids-dask-dependency; do
+    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
+done
+
+if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
+    sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" ${pyproject_file}
+    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
+fi
 
-cd python/
+cd ${package_dir}
 
 SKBUILD_CONFIGURE_OPTIONS="-DCUML_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DDISABLE_DEPRECATION_WARNINGS=ON -DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/" \
   python -m pip wheel . \

ci/release/update-version.sh

@@ -35,9 +35,10 @@ function sed_runner() {
 }
 
 
-# __init__.py and pyproject.toml versions
-sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cuml/__init__.py
-sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/pyproject.toml
+# Centralized version file update
+echo "${NEXT_FULL_TAG}" > VERSION
+
+# pyproject.toml versions
 sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pyproject.toml
 sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pyproject.toml
 sed_runner "s/pylibraft==.*\",/pylibraft==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pyproject.toml
@@ -72,6 +73,7 @@ DEPENDENCIES=(
   librmm
   pylibraft
   raft-dask
+  rapids-dask-dependency
   rmm
 )
 for FILE in dependencies.yaml conda/environments/*.yaml; do
@@ -80,17 +82,13 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do
   done
 done
 
-sed_runner "s|/branch-.*?/|/branch-${NEXT_SHORT_TAG}/|g" README.md
-sed_runner "s|/branch-.*?/|/branch-${NEXT_SHORT_TAG}/|g" python/README.md
+sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" README.md
+sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" python/README.md
+sed_runner "/- rapids-dask-dependency==/ s/==.*/==${NEXT_SHORT_TAG}\.*/g" python/README.md
 
 # Wheel builds clone cumlprims_mg, update its branch
 sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml
 
-# Wheel builds install dask-cuda from source, update its branch
-for FILE in .github/workflows/*.yaml; do
-  sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE};
-done
-
 # CI files
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"

ci/test_wheel.sh

@@ -12,9 +12,6 @@ if [[ "$(arch)" == "aarch64" ]]; then
     python -m pip install cmake
 fi
 
-# Always install latest dask for testing
-python -m pip install git+https://github.com/dask/[email protected] git+https://github.com/dask/[email protected] git+https://github.com/rapidsai/[email protected]
-
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install $(echo ./dist/cuml*.whl)[test]
 

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -16,12 +16,9 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core==2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
 - dask-ml
-- dask==2023.9.2
-- distributed==2023.9.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - gmock>=1.13.0
@@ -53,7 +50,7 @@ dependencies:
 - numpydoc
 - nvcc_linux-64=11.8
 - pip
-- pydata-sphinx-theme
+- pydata-sphinx-theme!=0.14.2
 - pylibraft==23.12.*
 - pynndescent==0.5.8
 - pytest
@@ -63,6 +60,7 @@ dependencies:
 - pytest-xdist
 - python>=3.9,<3.11
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - rmm==23.12.*
 - scikit-build>=0.13.1
@@ -77,5 +75,5 @@ dependencies:
 - treelite==3.9.1
 - umap-learn==0.5.3
 - pip:
-  - git+https://github.com/dask/dask-glm@main
+  - dask-glm==0.3.0
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -18,12 +18,9 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core==2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
 - dask-ml
-- dask==2023.9.2
-- distributed==2023.9.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - gmock>=1.13.0
@@ -49,7 +46,7 @@ dependencies:
 - numba>=0.57
 - numpydoc
 - pip
-- pydata-sphinx-theme
+- pydata-sphinx-theme!=0.14.2
 - pylibraft==23.12.*
 - pynndescent==0.5.8
 - pytest
@@ -59,6 +56,7 @@ dependencies:
 - pytest-xdist
 - python>=3.9,<3.11
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - rmm==23.12.*
 - scikit-build>=0.13.1
@@ -73,5 +71,5 @@ dependencies:
 - treelite==3.9.1
 - umap-learn==0.5.3
 - pip:
-  - git+https://github.com/dask/dask-glm@main
+  - dask-glm==0.3.0
 name: all_cuda-120_arch-x86_64

conda/environments/clang_tidy_cuda-118_arch-x86_64.yaml

@@ -8,8 +8,8 @@ channels:
 - nvidia
 dependencies:
 - c-compiler
-- clang-tools==16.0.6
-- clang==16.0.6
+- clang-tools==15.0.7
+- clang==15.0.7
 - cmake>=3.26.4
 - cuda-version=11.8
 - cudatoolkit

conda/recipes/cuml-cpu/meta.yaml

@@ -2,7 +2,7 @@
 
 # Usage:
 #   conda build . -c conda-forge -c numba -c rapidsai -c pytorch
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v')  + environ.get('VERSION_SUFFIX', '') %}
 {% set py_version = environ['CONDA_PY'] %}
 {% set date_string = environ['RAPIDS_DATE_STRING'] %}
 
@@ -11,7 +11,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}

conda/recipes/cuml/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2018-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
@@ -12,7 +12,7 @@ package:
   version: {{ version }}
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   number: {{ GIT_DESCRIBE_NUMBER }}
@@ -76,15 +76,13 @@ requirements:
     - cudf ={{ minor_version }}
     - cupy >=12.0.0
     - dask-cudf ={{ minor_version }}
-    - dask ==2023.9.2
-    - dask-core==2023.9.2
-    - distributed ==2023.9.2
     - joblib >=0.11
     - libcuml ={{ version }}
     - libcumlprims ={{ minor_version }}
     - pylibraft ={{ minor_version }}
     - python x.x
     - raft-dask ={{ minor_version }}
+    - rapids-dask-dependency ={{ minor_version }}
     - treelite {{ treelite_version }}
 
 tests:

conda/recipes/libcuml/meta.yaml

@@ -1,6 +1,6 @@
 # Copyright (c) 2018-2023, NVIDIA CORPORATION.
 
-{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %}
 {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
 {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %}
 {% set cuda_major = cuda_version.split('.')[0] %}
@@ -11,7 +11,7 @@ package:
   name: libcuml-split
 
 source:
-  git_url: ../../..
+  path: ../../..
 
 build:
   ignore_run_exports_from:

cpp/scripts/run-clang-tidy.py

@@ -25,7 +25,7 @@
 
 import tomli
 
-EXPECTED_VERSION = "16.0.6"
+EXPECTED_VERSION = "15.0.7"
 VERSION_REGEX = re.compile(r"  LLVM version ([0-9.]+)")
 GPU_ARCH_REGEX = re.compile(r"sm_(\d+)")
 SPACES = re.compile(r"\s+")

cpp/src/svm/kernelcache.cuh

@@ -130,8 +130,16 @@ class BatchCache : public raft::cache::Cache<math_t> {
     RAFT_CUDA_TRY(cudaMemsetAsync(tmp_buffer, 0, n_ws * 2 * sizeof(int), stream));
 
     // Init cub buffers
-    cub::DeviceRadixSort::SortKeys(
-      NULL, d_temp_storage_size, tmp_buffer, tmp_buffer, n_ws, 0, sizeof(int) * 8, stream);
+    cub::DeviceRadixSort::SortPairs(NULL,
+                                    d_temp_storage_size,
+                                    tmp_buffer,
+                                    tmp_buffer,
+                                    tmp_buffer,
+                                    tmp_buffer,
+                                    n_ws,
+                                    0,
+                                    sizeof(int) * 8,
+                                    stream);
     d_temp_storage.resize(d_temp_storage_size, stream);
   }