Consolidate C++ conda recipes and add libcugraph-tests package (#2124)

This PR includes the following changes: - Adds a `libcugraph-tests` package to the `libcugraph` recipe - This is a prerequisite for removing "Project Flash" from our build/CI scripts - The `libcugraph-tests` package was added as an additional output to the existing `libcugraph` recipe (which was renamed to `libcugraph-split`) - Consolidates remaining C++ recipes into `libcugraph-split` recipe - This gets rid of a lot of duplicate code between the recipes and reduces the number of times we have to call conda build in our CI scripts Authors: - Jordan Jacobelli (https://github.com/Ethyling) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) URL: #2124
rapidsai · Apr 6, 2022 · 1e46f7b · 1e46f7b
1 parent af7e935
commit 1e46f7b
Show file tree

Hide file tree

Showing 12 changed files with 120 additions and 138 deletions.
diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
@@ -36,10 +36,6 @@ export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -91,10 +87,8 @@ if [ "$BUILD_LIBCUGRAPH" == '1' ]; then
   gpuci_logger "Building conda package for libcugraph and libcugraph_etl"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph_etl
   else
     gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libcugraph
-    gpuci_conda_retry build --no-build-id --croot ${CONDA_BLD_DIR} --dirty --no-remove-work-dir conda/recipes/libcugraph_etl
     mkdir -p ${CONDA_BLD_DIR}/libcugraph
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/libcugraph/work
   fi

diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018-2021, NVIDIA CORPORATION.
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 #
 # Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
 
@@ -24,41 +24,25 @@ if [ -z "$MY_UPLOAD_KEY" ]; then
   return 0
 fi
 
-################################################################################
-# SETUP - Get conda file output locations
-################################################################################
-
-gpuci_logger "Get conda file output locations"
-
-export LIBCUGRAPH_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph --output`
-export LIBCUGRAPH_ETL_FILE=`conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph_etl --output`
-export PYLIBCUGRAPH_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/pylibcugraph --python=$PYTHON --output`
-export CUGRAPH_FILE=`conda build --croot ${CONDA_BLD_DIR} conda/recipes/cugraph --python=$PYTHON --output`
-
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
 gpuci_logger "Starting conda uploads"
 
 if [[ "$BUILD_LIBCUGRAPH" == "1" && "$UPLOAD_LIBCUGRAPH" == "1" ]]; then
-  test -e ${LIBCUGRAPH_FILE}
+  LIBCUGRAPH_FILES=$(conda build --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcugraph --output)
   echo "Upload libcugraph"
-  echo ${LIBCUGRAPH_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUGRAPH_FILE} --no-progress
-  test -e ${LIBCUGRAPH_ETL_FILE}
-  echo "Upload libcugraph_etl"
-  echo ${LIBCUGRAPH_ETL_FILE}
-  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${LIBCUGRAPH_ETL_FILE} --no-progress
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress ${LIBCUGRAPH_FILES}
 fi
 
 if [[ "$BUILD_CUGRAPH" == "1" ]]; then
+  PYLIBCUGRAPH_FILE=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/pylibcugraph --python=$PYTHON --output)
   test -e ${PYLIBCUGRAPH_FILE}
-  echo "Upload pylibcugraph"
-  echo ${PYLIBCUGRAPH_FILE}
+  echo "Upload pylibcugraph file: ${PYLIBCUGRAPH_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${PYLIBCUGRAPH_FILE} --no-progress
+  CUGRAPH_FILE=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/cugraph --python=$PYTHON --output)
   test -e ${CUGRAPH_FILE}
-  echo "Upload cugraph"
-  echo ${CUGRAPH_FILE}
+  echo "Upload cugraph file: ${CUGRAPH_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUGRAPH_FILE} --no-progress
 fi
diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
@@ -42,10 +42,6 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 # ucx-py version
 export UCX_PY_VERSION='0.26.*'
 
-export CMAKE_CUDA_COMPILER_LAUNCHER="sccache"
-export CMAKE_CXX_COMPILER_LAUNCHER="sccache"
-export CMAKE_C_COMPILER_LAUNCHER="sccache"
-
 ################################################################################
 # SETUP - Check environment
 ################################################################################
@@ -106,11 +102,8 @@ if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_logger "Build from source"
     $WORKSPACE/build.sh -v clean libcugraph pylibcugraph cugraph
 else
-    CONDA_FILE=`find ${CONDA_ARTIFACT_PATH} -name "libcugraph*.tar.bz2"`
-    CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
-    CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
-    echo "Installing $CONDA_FILE"
-    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} "$CONDA_FILE"
+    echo "Installing libcugraph-tests"
+    gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcugraph libcugraph_etl libcugraph-tests
 
     gpuci_logger "Install the master version of dask and distributed"
     pip install "git+https://github.com/dask/distributed.git" --upgrade --no-deps

diff --git a/ci/test.sh b/ci/test.sh
@@ -56,13 +56,6 @@ fi
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     cd ${CUGRAPH_ROOT}/cpp/build
-else
-    # ...cugraph/cpu/conda_work/... is the dir name when only 1 lib* library is
-    # present. For multiple libs (ie. libcugraph and libcugraph_etl), the
-    # "_work" dir is prefixed with the lib name.
-    export LIBCUGRAPH_BUILD_DIR="$WORKSPACE/ci/artifacts/cugraph/cpu/libcugraph_work/cpp/build"
-    export LD_LIBRARY_PATH="$LIBCUGRAPH_BUILD_DIR:$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
-    cd $LIBCUGRAPH_BUILD_DIR
 fi
 
 # Do not abort the script on error from this point on. This allows all tests to
@@ -74,7 +67,7 @@ if (python ${CUGRAPH_ROOT}/ci/utils/is_pascal.py); then
     echo "WARNING: skipping C++ tests on Pascal GPU arch."
 else
     echo "C++ gtests for cuGraph (single-GPU only)..."
-    for gt in $(find ./tests -name "*_TEST" | grep -v "MG_\|CAPI_" || true); do
+    for gt in "${CONDA_PREFIX}/bin/gtests/libcugraph/"*_TEST; do
         test_name=$(basename $gt)
         echo "Running gtest $test_name"
         ${gt} ${GTEST_FILTER} ${GTEST_ARGS}
@@ -83,7 +76,7 @@ else
     # FIXME: the C API tests do not generate XML, so CI systems will not show
     # them in the GUI. Failing C API tests will still fail CI though, and the
     # output will appear in logs.
-    for ct in $(find ./tests -name "CAPI_*_TEST"); do
+    for ct in "${CONDA_PREFIX}/bin/gtests/libcugraph_c/"CAPI_*_TEST; do
         test_name=$(basename $ct)
         echo "Running C API test $test_name"
         ${ct}

diff --git a/conda/recipes/libcugraph/build.sh b/conda/recipes/libcugraph/build.sh
@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
-
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 
 # This assumes the script is executed from the root of the repo directory
 
@@ -9,4 +8,4 @@
 # openmpi build dependencies). The conda package does NOT include these test
 # binaries or extra dependencies, but these are built here for use in CI runs.
 
-./build.sh libcugraph cpp-mgtests -v --allgpuarch
+./build.sh libcugraph libcugraph_etl cpp-mgtests -n -v --allgpuarch
diff --git a/conda/recipes/libcugraph/conda_build_config.yaml b/conda/recipes/libcugraph/conda_build_config.yaml
@@ -0,0 +1,17 @@
+cmake_version:
+  - ">=3.20.1,<3.23"
+
+doxygen_version:
+  - ">=1.8.11"
+
+boost_cpp_version:
+  - ">=1.66"
+
+nccl_version:
+  - ">=2.9.9"
+
+gtest_version:
+  - "=1.10.0"
+
+libcusolver_version:
+  - ">=11.2.1"
diff --git a/conda/recipes/libcugraph/install_libcugraph.sh b/conda/recipes/libcugraph/install_libcugraph.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build
diff --git a/conda/recipes/libcugraph/install_libcugraph_etl.sh b/conda/recipes/libcugraph/install_libcugraph_etl.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/libcugraph_etl/build
diff --git a/conda/recipes/libcugraph/install_libcugraph_tests.sh b/conda/recipes/libcugraph/install_libcugraph_tests.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+cmake --install cpp/build --component testing
+cmake --install cpp/libcugraph_etl/build --component testing
diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml
@@ -3,19 +3,18 @@
 # Usage:
 #   conda build -c nvidia -c rapidsai -c conda-forge .
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
-{% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
-{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %}
-{% set cuda_major=cuda_version.split('.')[0] %}
+{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
+{% set cuda_version = '.'.join(environ.get('CUDA', '11.5').split('.')[:2]) %}
+{% set cuda_major = cuda_version.split('.')[0] %}
+{% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0
+
 package:
-  name: libcugraph
-  version: {{ version }}
+  name: libcugraph-split
 
 source:
   git_url: ../../..
 
 build:
-  number: {{ GIT_DESCRIBE_NUMBER }}
-  string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
   script_env:
     - CC
     - CXX
@@ -34,29 +33,80 @@ build:
 
 requirements:
   build:
-    - cmake>=3.20.1,<3.23
-    - doxygen>=1.8.11
+    - cmake {{ cmake_version }}
+  host:
+    - doxygen {{ doxygen_version }}
     - cudatoolkit {{ cuda_version }}.*
-    - libraft-headers {{ minor_version }}
+    - libraft-headers {{ minor_version }}.*
     - libcugraphops {{ minor_version }}.*
     - librmm {{ minor_version }}.*
     - cudf {{ minor_version }}.*
-    - boost-cpp>=1.66
-    - nccl>=2.9.9
-    - ucx-proc=*=gpu
-    - gtest=1.10.0  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
-    - gmock=1.10.0  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
-  run:
-    - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
-    - libraft-headers {{ minor_version }}
-    - librmm {{ minor_version }}
-    - nccl>=2.9.9
+    - boost-cpp {{ boost_cpp_version }}
+    - nccl {{ nccl_version }}
     - ucx-proc=*=gpu
-    - libcugraphops {{ minor_version }}.*
-    - libcusolver>=11.2.1
+    - gtest {{ gtest_version }}  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
+    - gmock {{ gtest_version }}  # FIXME: pinned to version in https://github.com/rapidsai/integration/blob/branch-22.04/conda/recipes/versions.yaml
 
-about:
-  home: http://rapids.ai/
-  license: Apache-2.0
-  license_file: ../../../LICENSE
-  summary: libcugraph library
+outputs:
+  - name: libcugraph
+    version: {{ version }}
+    script: install_libcugraph.sh
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    requirements:
+      build:
+        - cmake {{ cmake_version }}
+      run:
+        - cudatoolkit {{ cuda_spec }}
+        - libraft-headers {{ minor_version }}
+        - librmm {{ minor_version }}
+        - nccl {{ nccl_version }}
+        - ucx-proc=*=gpu
+        - libcugraphops {{ minor_version }}
+        - libcusolver {{ libcusolver_version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_file: ../../../LICENSE
+      summary: libcugraph library
+  - name: libcugraph_etl
+    version: {{ version }}
+    script: install_libcugraph_etl.sh
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    requirements:
+      build:
+        - cmake {{ cmake_version }}
+      run:
+        - {{ pin_subpackage('libcugraph', exact=True) }}
+        - cudatoolkit {{ cuda_spec }}
+        - libcudf {{ minor_version }}
+        - libraft-headers {{ minor_version }}
+        - librmm {{ minor_version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_file: ../../../LICENSE
+      summary: libcugraph_etl library
+  - name: libcugraph-tests
+    version: {{ version }}
+    script: install_libcugraph_tests.sh
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+    requirements:
+      build:
+        - cmake {{ cmake_version }}
+      run:
+        - {{ pin_subpackage('libcugraph', exact=True) }}
+        - {{ pin_subpackage('libcugraph_etl', exact=True) }}
+        - cudatoolkit {{ cuda_spec }}
+        - gtest {{ gtest_version }}
+        - gmock {{ gtest_version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_file: ../../../LICENSE
+      summary: libcugraph test & benchmark executables
diff --git a/conda/recipes/libcugraph_etl/build.sh b/conda/recipes/libcugraph_etl/build.sh
diff --git a/conda/recipes/libcugraph_etl/meta.yaml b/conda/recipes/libcugraph_etl/meta.yaml