From 63427d5ab84830c385c46a8352f0dd4bfc846f60 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 6 Jun 2022 12:11:27 -0700 Subject: [PATCH 01/17] unpin dask --- ci/benchmark/build.sh | 6 +++--- ci/gpu/build.sh | 6 +++--- conda/environments/cudf_dev_cuda11.5.yml | 4 ++-- conda/recipes/custreamz/meta.yaml | 4 ++-- conda/recipes/dask-cudf/meta.yaml | 8 ++++---- python/dask_cudf/setup.py | 4 ++-- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh index a588fc6b161..9538826ba80 100755 --- a/ci/benchmark/build.sh +++ b/ci/benchmark/build.sh @@ -37,7 +37,7 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" # Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=0 +export INSTALL_DASK_MAIN=1 function remove_libcudf_kernel_cache_dir { EXITCODE=$? @@ -82,8 +82,8 @@ if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then gpuci_logger "gpuci_mamba_retry update dask" gpuci_mamba_retry update dask else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask==2022.05.2 conda-forge::distributed==2022.05.2 conda-forge::dask-core==2022.05.2 --force-reinstall" - gpuci_mamba_retry install conda-forge::dask==2022.05.2 conda-forge::distributed==2022.05.2 conda-forge::dask-core==2022.05.2 --force-reinstall + gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.05.2 conda-forge::distributed>=2022.05.2 conda-forge::dask-core>=2022.05.2 --force-reinstall" + gpuci_mamba_retry install conda-forge::dask>=2022.05.2 conda-forge::distributed>=2022.05.2 conda-forge::dask-core>=2022.05.2 --force-reinstall fi # Install the master version of streamz diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 8b8265075b9..40a4c5bb59f 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -32,7 +32,7 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` unset GIT_DESCRIBE_TAG # Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=0 +export INSTALL_DASK_MAIN=1 # ucx-py version export UCX_PY_VERSION='0.27.*' @@ -94,8 +94,8 @@ function install_dask { gpuci_mamba_retry update dask conda list else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask==2022.05.2 conda-forge::distributed==2022.05.2 conda-forge::dask-core==2022.05.2 --force-reinstall" - gpuci_mamba_retry install conda-forge::dask==2022.05.2 conda-forge::distributed==2022.05.2 conda-forge::dask-core==2022.05.2 --force-reinstall + gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.05.2 conda-forge::distributed>=2022.05.2 conda-forge::dask-core>=2022.05.2 --force-reinstall" + gpuci_mamba_retry install conda-forge::dask>=2022.05.2 conda-forge::distributed>=2022.05.2 conda-forge::dask-core>=2022.05.2 --force-reinstall fi # Install the main version of streamz gpuci_logger "Install the main version of streamz" diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index a00e103cca2..76d8dfe84e0 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -46,8 +46,8 @@ dependencies: - pydocstyle=6.1.1 - typing_extensions - pre-commit - - dask==2022.05.2 - - distributed==2022.05.2 + - dask>=2022.05.2 + - distributed>=2022.05.2 - streamz - arrow-cpp=7.0.0 - dlpack>=0.5,<0.6.0a0 diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index dcbd4410d8e..ce271d99247 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -29,8 +29,8 @@ requirements: - python - streamz - cudf {{ version }} - - dask==2022.05.2 - - distributed==2022.05.2 + - dask>=2022.05.2 + - distributed>=2022.05.2 - python-confluent-kafka >=1.7.0,<1.8.0a0 - cudf_kafka {{ version }} diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 99c2c26bfcb..d3c661f1492 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -24,14 +24,14 @@ requirements: host: - python - cudf {{ version }} - - dask==2022.05.2 - - distributed==2022.05.2 + - dask>=2022.05.2 + - distributed>=2022.05.2 - cudatoolkit {{ cuda_version }} run: - python - cudf {{ version }} - - dask==2022.05.2 - - distributed==2022.05.2 + - dask>=2022.05.2 + - distributed>=2022.05.2 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} test: # [linux64] diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index e9440467e98..2c32e8df377 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -10,8 +10,8 @@ install_requires = [ "cudf", - "dask==2022.05.2", - "distributed==2022.05.2", + "dask>=2022.05.2", + "distributed>=2022.05.2", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.5.0dev0", From 9d84d7b9250db16158ef02b26a0afc98bf2a1c51 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 6 Jun 2022 22:03:04 -0700 Subject: [PATCH 02/17] fix issues with latest dask and distributed --- python/dask_cudf/dask_cudf/backends.py | 14 ++++++++++---- .../dask_cudf/dask_cudf/tests/test_distributed.py | 4 +++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 1d3a15015fd..177a726c258 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -13,6 +13,7 @@ categorical_dtype_dispatch, concat_dispatch, group_split_dispatch, + grouper_dispatch, hash_object_dispatch, is_categorical_dtype_dispatch, make_meta_dispatch, @@ -296,12 +297,17 @@ def is_categorical_dtype_cudf(obj): return cudf.api.types.is_categorical_dtype(obj) +@grouper_dispatch.register((cudf.Series, cudf.DataFrame)) +def get_grouper_cudf(obj): + return cudf.core.groupby.Grouper + + try: - from dask.dataframe.dispatch import grouper_dispatch + from dask.dataframe.dispatch import pyarrow_schema_dispatch - @grouper_dispatch.register((cudf.Series, cudf.DataFrame)) - def get_grouper_cudf(obj): - return cudf.core.groupby.Grouper + @pyarrow_schema_dispatch.register((cudf.DataFrame, cudf.Series)) + def get_pyarrow_schema_cudf(obj): + return obj.to_arrow().schema except ImportError: pass diff --git a/python/dask_cudf/dask_cudf/tests/test_distributed.py b/python/dask_cudf/dask_cudf/tests/test_distributed.py index 876a66f78d7..564ad570f87 100644 --- a/python/dask_cudf/dask_cudf/tests/test_distributed.py +++ b/python/dask_cudf/dask_cudf/tests/test_distributed.py @@ -1,10 +1,12 @@ +# Copyright (c) 2020-2022, NVIDIA CORPORATION. + import numba.cuda import pytest import dask from dask import dataframe as dd from dask.distributed import Client -from distributed.utils_test import loop # noqa: F401 +from distributed.utils_test import cleanup, loop # noqa: F401 import cudf from cudf.testing._utils import assert_eq From 3f7bf0b556042611b81322727fbe42fa33f97ec9 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 7 Jun 2022 16:29:52 -0500 Subject: [PATCH 03/17] Update python/dask_cudf/dask_cudf/backends.py --- python/dask_cudf/dask_cudf/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 177a726c258..6055358efcc 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -305,7 +305,7 @@ def get_grouper_cudf(obj): try: from dask.dataframe.dispatch import pyarrow_schema_dispatch - @pyarrow_schema_dispatch.register((cudf.DataFrame, cudf.Series)) + @pyarrow_schema_dispatch.register((cudf.DataFrame, )) def get_pyarrow_schema_cudf(obj): return obj.to_arrow().schema From f5690dfcc425eb4a958e38a928b7666796d149a8 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 Jun 2022 16:33:33 -0700 Subject: [PATCH 04/17] style --- python/dask_cudf/dask_cudf/backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index 6055358efcc..6257fd6212a 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -305,7 +305,7 @@ def get_grouper_cudf(obj): try: from dask.dataframe.dispatch import pyarrow_schema_dispatch - @pyarrow_schema_dispatch.register((cudf.DataFrame, )) + @pyarrow_schema_dispatch.register((cudf.DataFrame,)) def get_pyarrow_schema_cudf(obj): return obj.to_arrow().schema From de3532f2d20e2346a505c3ada7d81220b9f9658b Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 8 Jun 2022 08:31:19 -0500 Subject: [PATCH 05/17] Update build.sh --- ci/cpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index f5ea2c902ef..8c2c5ab4fd8 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -3,7 +3,7 @@ ############################################## # cuDF CPU conda build script for CI # ############################################## -set -e +set -ex # Set path and build parallel level # FIXME: PATH variable shouldn't be necessary. From 7db7afdc12355d4a32ac712fa3539ffe190417c9 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 13 Jun 2022 10:59:39 -0700 Subject: [PATCH 06/17] test --- ci/cpu/build.sh | 1 + nvcc_linux-64_activate.sh | 106 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 nvcc_linux-64_activate.sh diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 8c2c5ab4fd8..d791921adf1 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -41,6 +41,7 @@ fi gpuci_logger "Check environment variables" env +cp -r /usr/local/src/conda/cudf-22.08.00a/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids diff --git a/nvcc_linux-64_activate.sh b/nvcc_linux-64_activate.sh new file mode 100644 index 00000000000..3dfaaa0f58f --- /dev/null +++ b/nvcc_linux-64_activate.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Backup environment variables (only if the variables are set) +if [[ ! -z "${CUDA_HOME+x}" ]] +then + export CUDA_HOME_CONDA_NVCC_BACKUP="${CUDA_HOME:-}" +fi + +if [[ ! -z "${CUDA_PATH+x}" ]] +then + export CUDA_PATH_CONDA_NVCC_BACKUP="${CUDA_PATH:-}" +fi + +if [[ ! -z "${CFLAGS+x}" ]] +then + export CFLAGS_CONDA_NVCC_BACKUP="${CFLAGS:-}" +fi + +if [[ ! -z "${CPPFLAGS+x}" ]] +then + export CPPFLAGS_CONDA_NVCC_BACKUP="${CPPFLAGS:-}" +fi + +if [[ ! -z "${CXXFLAGS+x}" ]] +then + export CXXFLAGS_CONDA_NVCC_BACKUP="${CXXFLAGS:-}" +fi + +if [[ ! -z "${CMAKE_ARGS+x}" ]] +then + export CMAKE_ARGS_CONDA_NVCC_BACKUP="${CMAKE_ARGS:-}" +fi + +# Default to using $(cuda-gdb) to specify $(CUDA_HOME). +if [[ -z "${CUDA_HOME+x}" ]] +then + CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) + if [[ -n "$CUDA_GDB_EXECUTABLE" ]] + then + CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) + else + echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" + return 1 + fi +fi + +if [[ ! -d "${CUDA_HOME}" ]] +then + echo "Directory specified in CUDA_HOME(=${CUDA_HOME}) doesn't exist" + return 1 +fi + +if [[ ! -f "${CUDA_HOME}/lib64/stubs/libcuda.so" ]] +then + echo "File ${CUDA_HOME}/lib64/stubs/libcuda.so doesn't exist" + return 1 +fi + +if [[ -z "$(${CUDA_HOME}/bin/nvcc --version | grep "Cuda compilation tools, release 11.5")" ]] +then + echo "Version of installed CUDA didn't match package" + return 1 +fi + +export CUDA_HOME="${CUDA_HOME}" +export CFLAGS="${CFLAGS} -isystem ${CUDA_HOME}/include" +export CPPFLAGS="${CPPFLAGS} -isystem ${CUDA_HOME}/include" +export CXXFLAGS="${CXXFLAGS} -isystem ${CUDA_HOME}/include" + +### CMake configurations + +# CMake looks up components in CUDA_PATH, not CUDA_HOME +export CUDA_PATH="${CUDA_HOME}" +# New-style CUDA integrations in CMake +CMAKE_ARGS="${CMAKE_ARGS:-} -DCUDAToolkit_ROOT=${CUDA_HOME}" +# Old-style CUDA integrations in CMake +## See https://github.com/conda-forge/nvcc-feedstock/pull/58#issuecomment-752179349 +CMAKE_ARGS+=" -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME}" +## Avoid https://github.com/conda-forge/openmm-feedstock/pull/44#issuecomment-753560234 +## We need CUDA_HOME in _front_ of CMAKE_FIND_ROOT_PATH +CMAKE_ARGS="$(echo ${CMAKE_ARGS} | sed -E -e "s|(-DCMAKE_FIND_ROOT_PATH=)(\S+)|\1$CUDA_HOME;\2|")" +export CMAKE_ARGS="${CMAKE_ARGS}" + +### /CMake configurations + +mkdir -p "${CONDA_BUILD_SYSROOT}/lib" +mkdir -p "${CONDA_PREFIX}/lib/stubs" + +# Add $(libcuda.so) shared object stub to the compiler sysroot. +# Needed for things that want to link to $(libcuda.so). +# Stub is used to avoid getting driver code linked into binaries. + + +if [[ "${CONDA_BUILD}" == 1 ]] +then + # Make a backup of $(libcuda.so) if it exists + if [[ -f "${CONDA_BUILD_SYSROOT}/lib/libcuda.so" ]] + then + LIBCUDA_SO_CONDA_NVCC_BACKUP="${CONDA_BUILD_SYSROOT}/lib/libcuda.so-conda-nvcc-backup" + mv "${CONDA_BUILD_SYSROOT}/lib/libcuda.so" "${LIBCUDA_SO_CONDA_NVCC_BACKUP}" + fi + ln -s "${CUDA_HOME}/lib64/stubs/libcuda.so" "${CONDA_BUILD_SYSROOT}/lib/libcuda.so" +else + ln -sf "${CUDA_HOME}/lib64/stubs/libcuda.so" "${CONDA_PREFIX}/lib/stubs/libcuda.so" +fi + From 08d612dd9533b0026e0641af5d62eee95a9a8521 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 13 Jun 2022 11:08:59 -0700 Subject: [PATCH 07/17] temp --- ci/cpu/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index d791921adf1..21548552586 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -41,7 +41,8 @@ fi gpuci_logger "Check environment variables" env -cp -r /usr/local/src/conda/cudf-22.08.00a/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh +ls $WORKSPACE +cp -r $WORKSPACE/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids From 368d138cb0dfda12a1b46bf86f397d34c6b9ba7c Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 13 Jun 2022 13:16:09 -0700 Subject: [PATCH 08/17] temp --- conda/recipes/cudf/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index bcd9b81f966..05327b0f337 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -67,6 +67,7 @@ requirements: test: # [linux64] requires: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] + - {{ compiler('cuda') }} {{ cuda_version }} imports: # [linux64] - cudf # [linux64] From 791e34b059668cffa8243bf15cdb1023f3322235 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Jun 2022 16:21:15 -0500 Subject: [PATCH 09/17] Update build.sh --- ci/cpu/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 21548552586..e2e2a0bf61c 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -41,8 +41,8 @@ fi gpuci_logger "Check environment variables" env -ls $WORKSPACE -cp -r $WORKSPACE/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh +#ls $WORKSPACE +#cp -r $WORKSPACE/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids From 9c778d030f1da1b03f09a82a0e99147b899865fa Mon Sep 17 00:00:00 2001 From: jakirkham Date: Mon, 13 Jun 2022 15:45:35 -0700 Subject: [PATCH 10/17] Add C/C++ compilers to tests Should ensure `CONDA_BUILD_SYSROOT` is set. --- conda/recipes/cudf/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 05327b0f337..7da03ae8fa7 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -67,6 +67,8 @@ requirements: test: # [linux64] requires: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] + - {{ compiler('c') }} + - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} imports: # [linux64] - cudf # [linux64] From dc9039a97268635b35b9e4d2f54a2a97314f19d7 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Jun 2022 18:43:49 -0500 Subject: [PATCH 11/17] Update ci/cpu/build.sh --- ci/cpu/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index e2e2a0bf61c..21548552586 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -41,8 +41,8 @@ fi gpuci_logger "Check environment variables" env -#ls $WORKSPACE -#cp -r $WORKSPACE/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh +ls $WORKSPACE +cp -r $WORKSPACE/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids From 0d108cd8f5371659de0ef3d61d1ef831a06d49dc Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 14 Jun 2022 15:25:41 -0500 Subject: [PATCH 12/17] Update meta.yaml --- conda/recipes/cudf/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 7da03ae8fa7..c629727947a 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -22,8 +22,8 @@ build: # libcudf's run_exports pinning is looser than we would like ignore_run_exports: - libcudf - ignore_run_exports_from: - - {{ compiler('cuda') }} +# ignore_run_exports_from: +# - {{ compiler('cuda') }} requirements: build: From c6c01cc9b386cdfba143c7582fbf3f87f991fc73 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 15 Jun 2022 12:50:24 -0500 Subject: [PATCH 13/17] Update build.sh --- ci/cpu/build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 21548552586..8c2c5ab4fd8 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -41,8 +41,6 @@ fi gpuci_logger "Check environment variables" env -ls $WORKSPACE -cp -r $WORKSPACE/nvcc_linux-64_activate.sh /opt/conda/envs/rapids/etc/conda/activate.d/nvcc_linux-64_activate.sh gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids From b1dc672755c6297da3b2b465c2cc28b0660f51b5 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 15 Jun 2022 12:50:57 -0500 Subject: [PATCH 14/17] Update meta.yaml --- conda/recipes/cudf/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index c629727947a..7da03ae8fa7 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -22,8 +22,8 @@ build: # libcudf's run_exports pinning is looser than we would like ignore_run_exports: - libcudf -# ignore_run_exports_from: -# - {{ compiler('cuda') }} + ignore_run_exports_from: + - {{ compiler('cuda') }} requirements: build: From b87a29587fdd14093e5faa890b667eb13c584243 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 15 Jun 2022 12:53:23 -0500 Subject: [PATCH 15/17] Update meta.yaml --- conda/recipes/cudf/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 7da03ae8fa7..e12b5c3b219 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -67,9 +67,9 @@ requirements: test: # [linux64] requires: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler('cuda') }} {{ cuda_version }} +# - {{ compiler('c') }} +# - {{ compiler('cxx') }} +# - {{ compiler('cuda') }} {{ cuda_version }} imports: # [linux64] - cudf # [linux64] From 2e7c3b4761ef479f784beebb3f0902bf8d863c92 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 15 Jun 2022 13:04:14 -0700 Subject: [PATCH 16/17] revert --- ci/cpu/build.sh | 2 +- conda/recipes/cudf/meta.yaml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 8c2c5ab4fd8..f5ea2c902ef 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -3,7 +3,7 @@ ############################################## # cuDF CPU conda build script for CI # ############################################## -set -ex +set -e # Set path and build parallel level # FIXME: PATH variable shouldn't be necessary. diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 016e0684d43..e76855a7d13 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -67,9 +67,6 @@ requirements: test: # [linux64] requires: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] -# - {{ compiler('c') }} -# - {{ compiler('cxx') }} -# - {{ compiler('cuda') }} {{ cuda_version }} imports: # [linux64] - cudf # [linux64] From 401748f99dc91518ab1b90b5dc8fef7e05e9bbbe Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 15 Jun 2022 13:04:28 -0700 Subject: [PATCH 17/17] revert --- nvcc_linux-64_activate.sh | 106 -------------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 nvcc_linux-64_activate.sh diff --git a/nvcc_linux-64_activate.sh b/nvcc_linux-64_activate.sh deleted file mode 100644 index 3dfaaa0f58f..00000000000 --- a/nvcc_linux-64_activate.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -# Backup environment variables (only if the variables are set) -if [[ ! -z "${CUDA_HOME+x}" ]] -then - export CUDA_HOME_CONDA_NVCC_BACKUP="${CUDA_HOME:-}" -fi - -if [[ ! -z "${CUDA_PATH+x}" ]] -then - export CUDA_PATH_CONDA_NVCC_BACKUP="${CUDA_PATH:-}" -fi - -if [[ ! -z "${CFLAGS+x}" ]] -then - export CFLAGS_CONDA_NVCC_BACKUP="${CFLAGS:-}" -fi - -if [[ ! -z "${CPPFLAGS+x}" ]] -then - export CPPFLAGS_CONDA_NVCC_BACKUP="${CPPFLAGS:-}" -fi - -if [[ ! -z "${CXXFLAGS+x}" ]] -then - export CXXFLAGS_CONDA_NVCC_BACKUP="${CXXFLAGS:-}" -fi - -if [[ ! -z "${CMAKE_ARGS+x}" ]] -then - export CMAKE_ARGS_CONDA_NVCC_BACKUP="${CMAKE_ARGS:-}" -fi - -# Default to using $(cuda-gdb) to specify $(CUDA_HOME). -if [[ -z "${CUDA_HOME+x}" ]] -then - CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) - if [[ -n "$CUDA_GDB_EXECUTABLE" ]] - then - CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) - else - echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" - return 1 - fi -fi - -if [[ ! -d "${CUDA_HOME}" ]] -then - echo "Directory specified in CUDA_HOME(=${CUDA_HOME}) doesn't exist" - return 1 -fi - -if [[ ! -f "${CUDA_HOME}/lib64/stubs/libcuda.so" ]] -then - echo "File ${CUDA_HOME}/lib64/stubs/libcuda.so doesn't exist" - return 1 -fi - -if [[ -z "$(${CUDA_HOME}/bin/nvcc --version | grep "Cuda compilation tools, release 11.5")" ]] -then - echo "Version of installed CUDA didn't match package" - return 1 -fi - -export CUDA_HOME="${CUDA_HOME}" -export CFLAGS="${CFLAGS} -isystem ${CUDA_HOME}/include" -export CPPFLAGS="${CPPFLAGS} -isystem ${CUDA_HOME}/include" -export CXXFLAGS="${CXXFLAGS} -isystem ${CUDA_HOME}/include" - -### CMake configurations - -# CMake looks up components in CUDA_PATH, not CUDA_HOME -export CUDA_PATH="${CUDA_HOME}" -# New-style CUDA integrations in CMake -CMAKE_ARGS="${CMAKE_ARGS:-} -DCUDAToolkit_ROOT=${CUDA_HOME}" -# Old-style CUDA integrations in CMake -## See https://github.com/conda-forge/nvcc-feedstock/pull/58#issuecomment-752179349 -CMAKE_ARGS+=" -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME}" -## Avoid https://github.com/conda-forge/openmm-feedstock/pull/44#issuecomment-753560234 -## We need CUDA_HOME in _front_ of CMAKE_FIND_ROOT_PATH -CMAKE_ARGS="$(echo ${CMAKE_ARGS} | sed -E -e "s|(-DCMAKE_FIND_ROOT_PATH=)(\S+)|\1$CUDA_HOME;\2|")" -export CMAKE_ARGS="${CMAKE_ARGS}" - -### /CMake configurations - -mkdir -p "${CONDA_BUILD_SYSROOT}/lib" -mkdir -p "${CONDA_PREFIX}/lib/stubs" - -# Add $(libcuda.so) shared object stub to the compiler sysroot. -# Needed for things that want to link to $(libcuda.so). -# Stub is used to avoid getting driver code linked into binaries. - - -if [[ "${CONDA_BUILD}" == 1 ]] -then - # Make a backup of $(libcuda.so) if it exists - if [[ -f "${CONDA_BUILD_SYSROOT}/lib/libcuda.so" ]] - then - LIBCUDA_SO_CONDA_NVCC_BACKUP="${CONDA_BUILD_SYSROOT}/lib/libcuda.so-conda-nvcc-backup" - mv "${CONDA_BUILD_SYSROOT}/lib/libcuda.so" "${LIBCUDA_SO_CONDA_NVCC_BACKUP}" - fi - ln -s "${CUDA_HOME}/lib64/stubs/libcuda.so" "${CONDA_BUILD_SYSROOT}/lib/libcuda.so" -else - ln -sf "${CUDA_HOME}/lib64/stubs/libcuda.so" "${CONDA_PREFIX}/lib/stubs/libcuda.so" -fi -