S3 support

conda/environments/all_cuda-118_arch-aarch64.yaml

@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-python>=11.7.1,<12.0a0
@@ -17,6 +19,7 @@ dependencies:
 - dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-python>=11.7.1,<12.0a0
@@ -19,6 +21,7 @@ dependencies:
 - gcc_linux-64=11.*
 - libcufile-dev=1.4.0.31
 - libcufile=1.4.0.31
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

conda/environments/all_cuda-125_arch-aarch64.yaml

@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-nvcc
@@ -18,6 +20,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
 - libcufile-dev
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -6,6 +6,8 @@ channels:
 - conda-forge
 - nvidia
 dependencies:
+- aws-sdk-cpp
+- boto3>=1.21.21
 - c-compiler
 - cmake>=3.26.4,!=3.30.0
 - cuda-nvcc
@@ -18,6 +20,7 @@ dependencies:
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - libcufile-dev
+- moto>=4.0.8
 - ninja
 - numcodecs <0.12.0
 - numpy>=1.23,<2.0a0

conda/recipes/kvikio/meta.yaml

@@ -52,6 +52,7 @@ requirements:
     - {{ compiler('cuda') }}
     {% endif %}
     - {{ stdlib("c") }}
+    - aws-sdk-cpp
   host:
     - python
     - pip
@@ -64,11 +65,13 @@ requirements:
     - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - scikit-build-core >=0.10.0
     - libkvikio ={{ version }}
+    - aws-sdk-cpp
   run:
     - python
     - numpy >=1.23,<2.0a0
     - cupy >=12.0.0
     - zarr
+    - aws-sdk-cpp
     # See https://github.com/zarr-developers/numcodecs/pull/475
     - numcodecs <0.12.0
     - packaging

conda/recipes/libkvikio/meta.yaml

@@ -43,6 +43,7 @@ requirements:
     {% endif %}
     - ninja
     - {{ stdlib("c") }}
+    - aws-sdk-cpp
   host:
     - cuda-version ={{ cuda_version }}
     {% if cuda_major == "11" %}
@@ -52,6 +53,7 @@ requirements:
     {% else %}
     - libcufile-dev  # [linux]
     {% endif %}
+    - aws-sdk-cpp
 
 outputs:
   - name: libkvikio
@@ -72,8 +74,10 @@ outputs:
     requirements:
       build:
         - cmake {{ cmake_version }}
+        - aws-sdk-cpp
       host:
         - cuda-version ={{ cuda_version }}
+        - aws-sdk-cpp
       run:
         - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
         {% if cuda_major == "11" %}
@@ -83,6 +87,7 @@ outputs:
         {% else %}
         - libcufile-dev  # [linux]
         {% endif %}
+        - aws-sdk-cpp
     test:
         commands:
           - test -f $PREFIX/include/kvikio/file_handle.hpp
@@ -106,6 +111,7 @@ outputs:
         - cuda-cudart-dev
         - libcufile-dev  # [linux]
         {% endif %}
+        - aws-sdk-cpp
     requirements:
       build:
         - cmake {{ cmake_version }}
@@ -118,6 +124,7 @@ outputs:
         - cuda-cudart-dev
         - libcufile-dev  # [linux]
         {% endif %}
+        - aws-sdk-cpp
       run:
         - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
         {% if cuda_major == "11" %}
@@ -127,6 +134,7 @@ outputs:
         - cuda-cudart
         - libcufile  # [linux]
         {% endif %}
+        - aws-sdk-cpp
     about:
       home: https://rapids.ai
       license: Apache-2.0

cpp/CMakeLists.txt

@@ -55,6 +55,12 @@ rapids_find_package(
   INSTALL_EXPORT_SET kvikio-exports
 )
 
+rapids_find_package(
+  AWSSDK COMPONENTS s3
+  BUILD_EXPORT_SET kvikio-exports
+  INSTALL_EXPORT_SET kvikio-exports
+)
+
 rapids_find_package(
   cuFile
   BUILD_EXPORT_SET kvikio-exports
@@ -130,6 +136,7 @@ target_include_directories(
 )
 target_link_libraries(
   kvikio INTERFACE Threads::Threads ${CMAKE_DL_LIBS} nvtx3::nvtx3-cpp BS::thread_pool
+                   ${AWSSDK_LINK_LIBRARIES}
 )
 target_compile_features(kvikio INTERFACE cxx_std_17)
 

cpp/examples/CMakeLists.txt

@@ -14,6 +14,8 @@
 
 set(TEST_INSTALL_PATH bin/tests/libkvikio)
 
+# Example: basic_io
+
 if(CUDAToolkit_FOUND)
   add_executable(BASIC_IO_TEST basic_io.cpp)
   set_target_properties(BASIC_IO_TEST PROPERTIES INSTALL_RPATH "\$ORIGIN/../../lib")
@@ -35,6 +37,8 @@ else()
   message(STATUS "Cannot build the basic_io example when CUDA is not found")
 endif()
 
+# Example: basic_no_cuda
+
 add_executable(BASIC_NO_CUDA_TEST basic_no_cuda.cpp)
 set_target_properties(BASIC_NO_CUDA_TEST PROPERTIES INSTALL_RPATH "\$ORIGIN/../../lib")
 target_include_directories(BASIC_NO_CUDA_TEST PRIVATE ../include)

cpp/include/kvikio/file_handle.hpp

@@ -286,7 +286,7 @@ class FileHandle {
    *
    * @return The number of bytes
    */
-  [[nodiscard]] inline std::size_t nbytes() const
+  [[nodiscard]] std::size_t nbytes() const
   {
     if (closed()) { return 0; }
     if (_nbytes == 0) { _nbytes = detail::get_file_size(_fd_direct_off); }