Use nvcomp conda package. (#13566)

This PR uses conda-forge packages of `nvcomp` rather than fetching a tarball. This means that the nvcomp binary should not be shipped in the libcudf conda package, but is instead listed as a dependency. This will reduce libcudf's conda package size. Addresses part of #13230. Authors: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) - https://github.com/jakirkham URL: #13566
rapidsai · Jun 26, 2023 · aed7174 · aed7174
1 parent 042e0a3
commit aed7174
Show file tree

Hide file tree

Showing 8 changed files with 23 additions and 11 deletions.
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -56,6 +56,7 @@ dependencies:
 - numpy>=1.21
 - numpydoc
 - nvcc_linux-64=11.8
+- nvcomp==2.6.1
 - nvtx>=0.2.1
 - packaging
 - pandas>=1.3,<1.6.0dev0

diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml
@@ -55,6 +55,7 @@ dependencies:
 - numba>=0.57
 - numpy>=1.21
 - numpydoc
+- nvcomp==2.6.1
 - nvtx>=0.2.1
 - packaging
 - pandas>=1.3,<1.6.0dev0

diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml
@@ -37,6 +37,9 @@ fmt_version:
 spdlog_version:
   - ">=1.11.0,<1.12"
 
+nvcomp_version:
+  - "=2.6.1"
+
 # The CTK libraries below are missing from the conda-forge::cudatoolkit package
 # for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages
 # and the "*_run_*" version specifiers correspond to `11.x` packages.

diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml
@@ -64,6 +64,7 @@ requirements:
     - libcurand-dev
     {% endif %}
     - cuda-version ={{ cuda_version }}
+    - nvcomp {{ nvcomp_version }}
     - libarrow {{ libarrow_version }}
     - dlpack {{ dlpack_version }}
     - librdkafka {{ librdkafka_version }}
@@ -98,6 +99,7 @@ outputs:
         - libcufile  # [linux64]
         {% endif %}
         - cuda-version {{ cuda_spec }}
+        - nvcomp {{ nvcomp_version }}
         - librmm ={{ minor_version }}
         - libkvikio ={{ minor_version }}
         - libarrow {{ libarrow_version }}
@@ -352,8 +354,6 @@ outputs:
       license_family: APACHE
       license_file: LICENSE
       summary: libcudf library
-      prelink_message:
-        - nvcomp.txt
   - name: libcudf_kafka
     version: {{ version }}
     script: install_libcudf_kafka.sh

diff --git a/conda/recipes/libcudf/nvcomp.txt b/conda/recipes/libcudf/nvcomp.txt
diff --git a/conda/recipes/libcudf/post-link.sh b/conda/recipes/libcudf/post-link.sh
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -226,6 +226,20 @@ dependencies:
           - libarrow==11.0.0.*
           - librdkafka>=1.9.0,<1.10.0a0
           - spdlog>=1.11.0,<1.12
+    specific:
+      - output_types: conda
+        matrices:
+          - matrix:
+              arch: x86_64
+            packages:
+              # Align nvcomp version with rapids-cmake
+              # TODO: not yet available for aarch64 CUDA 12
+              - &nvcomp nvcomp==2.6.1
+          - matrix:
+              arch: aarch64
+              cuda: "11.8"
+            packages:
+              - *nvcomp
   build_wheels:
     common:
       - output_types: pyproject

diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake
@@ -12,6 +12,8 @@
 # the License.
 # =============================================================================
 if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake)
+  set(rapids-cmake-repo vyasr/rapids-cmake)
+  set(rapids-cmake-branch fix/nvcomp_find)
   file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.08/RAPIDS.cmake
        ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake
   )