From aed7174eae6c6eb38fbf186938df44f88787cf29 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 26 Jun 2023 15:29:41 -0500 Subject: [PATCH] Use nvcomp conda package. (#13566) This PR uses conda-forge packages of `nvcomp` rather than fetching a tarball. This means that the nvcomp binary should not be shipped in the libcudf conda package, but is instead listed as a dependency. This will reduce libcudf's conda package size. Addresses part of #13230. Authors: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/13566 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-120_arch-x86_64.yaml | 1 + conda/recipes/libcudf/conda_build_config.yaml | 3 +++ conda/recipes/libcudf/meta.yaml | 4 ++-- conda/recipes/libcudf/nvcomp.txt | 3 --- conda/recipes/libcudf/post-link.sh | 6 ------ dependencies.yaml | 14 ++++++++++++++ fetch_rapids.cmake | 2 ++ 8 files changed, 23 insertions(+), 11 deletions(-) delete mode 100644 conda/recipes/libcudf/nvcomp.txt delete mode 100644 conda/recipes/libcudf/post-link.sh diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2deb2959f85..e8ac871129e 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -56,6 +56,7 @@ dependencies: - numpy>=1.21 - numpydoc - nvcc_linux-64=11.8 +- nvcomp==2.6.1 - nvtx>=0.2.1 - packaging - pandas>=1.3,<1.6.0dev0 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index e49d853899c..17ee1548571 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -55,6 +55,7 @@ dependencies: - numba>=0.57 - numpy>=1.21 - numpydoc +- nvcomp==2.6.1 - nvtx>=0.2.1 - packaging - pandas>=1.3,<1.6.0dev0 diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 2a3927e1833..4d03dab2f35 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -37,6 +37,9 @@ fmt_version: spdlog_version: - ">=1.11.0,<1.12" +nvcomp_version: + - "=2.6.1" + # The CTK libraries below are missing from the conda-forge::cudatoolkit package # for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages # and the "*_run_*" version specifiers correspond to `11.x` packages. diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index c21ec70327e..4e9b5e2fdc1 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -64,6 +64,7 @@ requirements: - libcurand-dev {% endif %} - cuda-version ={{ cuda_version }} + - nvcomp {{ nvcomp_version }} - libarrow {{ libarrow_version }} - dlpack {{ dlpack_version }} - librdkafka {{ librdkafka_version }} @@ -98,6 +99,7 @@ outputs: - libcufile # [linux64] {% endif %} - cuda-version {{ cuda_spec }} + - nvcomp {{ nvcomp_version }} - librmm ={{ minor_version }} - libkvikio ={{ minor_version }} - libarrow {{ libarrow_version }} @@ -352,8 +354,6 @@ outputs: license_family: APACHE license_file: LICENSE summary: libcudf library - prelink_message: - - nvcomp.txt - name: libcudf_kafka version: {{ version }} script: install_libcudf_kafka.sh diff --git a/conda/recipes/libcudf/nvcomp.txt b/conda/recipes/libcudf/nvcomp.txt deleted file mode 100644 index 9a0047e71fa..00000000000 --- a/conda/recipes/libcudf/nvcomp.txt +++ /dev/null @@ -1,3 +0,0 @@ -By downloading and using the libcudf conda package, you accept the terms -and conditions of the NVIDIA NVCOMP Software License Agreement: - https://developer.download.nvidia.com/compute/nvcomp/2.3/LICENSE.txt diff --git a/conda/recipes/libcudf/post-link.sh b/conda/recipes/libcudf/post-link.sh deleted file mode 100644 index 8ae2349f791..00000000000 --- a/conda/recipes/libcudf/post-link.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# Only add the license notice to libcudf and not our examples / tests -if [[ "$PKG_NAME" == "libcudf" ]]; then - cat ./nvcomp.txt >> $PREFIX/.messages.txt -fi diff --git a/dependencies.yaml b/dependencies.yaml index 70d281704ce..1387a348501 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -226,6 +226,20 @@ dependencies: - libarrow==11.0.0.* - librdkafka>=1.9.0,<1.10.0a0 - spdlog>=1.11.0,<1.12 + specific: + - output_types: conda + matrices: + - matrix: + arch: x86_64 + packages: + # Align nvcomp version with rapids-cmake + # TODO: not yet available for aarch64 CUDA 12 + - &nvcomp nvcomp==2.6.1 + - matrix: + arch: aarch64 + cuda: "11.8" + packages: + - *nvcomp build_wheels: common: - output_types: pyproject diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake index 7ef42466e9f..b99428b0c93 100644 --- a/fetch_rapids.cmake +++ b/fetch_rapids.cmake @@ -12,6 +12,8 @@ # the License. # ============================================================================= if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake) + set(rapids-cmake-repo vyasr/rapids-cmake) + set(rapids-cmake-branch fix/nvcomp_find) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.08/RAPIDS.cmake ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake )