Build C++ XTensor project as Python package so users can pip install without any additional dependencies

[lionlai1989]

I am following XTensor-python tutorial to build a XTensor C++ project which has Python binding with CMake. The end goal is to create a self-contained Python package that users can install via pip install without requiring any additional dependencies.

The issue is that the target mymodule cannot link to xtensor::optimize but only xtensor.

Minimal Reproducible Example

Here is the minimal reproducible example: (You can follow the instruction here and reproduce the result)

Project Structure Explanation:

• Project structure:

.
├── CMakeLists.txt
├── extern
│   ├── pybind11
│   ├── xsimd
│   ├── xtensor
│   ├── xtensor-python
│   └── xtl
├── pyproject.toml
├── setup.py
└── src
    └── main.cpp

• extern/ folder: pybind11, xtl, xtensor, xsimd, and xtensor-python are all added as git submodules. The reason is that this project is a self-contained C++/Python project. Users can use only pip install to install it without installing anything other things manually.

• CMakeLists.txt: The content is copied and pasted from xtensor-python tutorial, except xtensor packages are not installed and found by find_package but with add_subdirectory (I will explain the commented code FetchContent later). The reason of not using find_package is that I must first install packages so find_package will work. But I can't install packages for users on users' computer so packages are added as submodules and added by using add_subdirectory.

cmake_minimum_required(VERSION 3.22)

project(mymodule)

# Must be submodule. Cannot be installed with pip
add_subdirectory(extern/pybind11)

# Use FetchContent
# include(FetchContent)
# # Download and configure xtl
# FetchContent_Declare(xtl
#     GIT_REPOSITORY https://github.com/xtensor-stack/xtl.git
#     GIT_TAG master
#     OVERRIDE_FIND_PACKAGE
# )
# FetchContent_MakeAvailable(xtl)
# # Download and configure xsimd
# FetchContent_Declare(xsimd
#     GIT_REPOSITORY https://github.com/xtensor-stack/xsimd.git
#     GIT_TAG master
#     OVERRIDE_FIND_PACKAGE
# )
# FetchContent_MakeAvailable(xsimd)
# # Download and configure xtensor
# FetchContent_Declare(xtensor
#     GIT_REPOSITORY https://github.com/xtensor-stack/xtensor.git
#     GIT_TAG master
#     OVERRIDE_FIND_PACKAGE
# )
# FetchContent_MakeAvailable(xtensor)

# Use add_subdirectory
add_subdirectory(extern/xtl)
add_subdirectory(extern/xtensor)
add_subdirectory(extern/xsimd)
add_subdirectory(extern/xtensor-python)

find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy)

pybind11_add_module(mymodule src/main.cpp)
target_link_libraries(mymodule PUBLIC xtensor pybind11::module xtensor-python Python::NumPy)

target_compile_definitions(mymodule PRIVATE VERSION_INFO=0.1.0)

• pyproject.toml: All required packages to build the system are added here.

[build-system]
requires = [
    "pip<=23.0",
    "setuptools>=62,<=65",
    "wheel",
    "ninja",
    "cmake>=3.22",
    "numpy"
]
build-backend = "setuptools.build_meta"

[project]
name = "mymodule"
version = "0.0.1"
requires-python = ">=3.8"

dependencies = [
    "numpy"
]

• setup.py: The content is from Pybind11/CMake tutorial on github

import os
import re
import subprocess
import sys
from pathlib import Path

from setuptools import Extension, setup
from setuptools.command.build_ext import build_ext

# Convert distutils Windows platform specifiers to CMake -A arguments
PLAT_TO_CMAKE = {
    "win32": "Win32",
    "win-amd64": "x64",
    "win-arm32": "ARM",
    "win-arm64": "ARM64",
}


# A CMakeExtension needs a sourcedir instead of a file list.
# The name must be the _single_ output extension from the CMake build.
# If you need multiple extensions, see scikit-build.
class CMakeExtension(Extension):
    def __init__(self, name: str, sourcedir: str = "") -> None:
        super().__init__(name, sources=[])
        self.sourcedir = os.fspath(Path(sourcedir).resolve())


class CMakeBuild(build_ext):
    def build_extension(self, ext: CMakeExtension) -> None:
        # Must be in this form due to bug in .resolve() only fixed in Python 3.10+
        ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name)  # type: ignore[no-untyped-call]
        extdir = ext_fullpath.parent.resolve()

        # Using this requires trailing slash for auto-detection & inclusion of
        # auxiliary "native" libs

        debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug
        cfg = "Debug" if debug else "Release"

        # CMake lets you override the generator - we need to check this.
        # Can be set with Conda-Build, for example.
        cmake_generator = os.environ.get("CMAKE_GENERATOR", "")

        # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON
        # EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code
        # from Python.
        cmake_args = [
            f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}",
            f"-DPYTHON_EXECUTABLE={sys.executable}",
            f"-DCMAKE_BUILD_TYPE={cfg}",  # not used on MSVC, but no harm
        ]
        build_args = []
        # Adding CMake arguments set as environment variable
        # (needed e.g. to build for ARM OSx on conda-forge)
        if "CMAKE_ARGS" in os.environ:
            cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item]

        # In this example, we pass in the version to C++. You might not need to.
        cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"]  # type: ignore[attr-defined]

        if self.compiler.compiler_type != "msvc":
            # Using Ninja-build since it a) is available as a wheel and b)
            # multithreads automatically. MSVC would require all variables be
            # exported for Ninja to pick it up, which is a little tricky to do.
            # Users can override the generator with CMAKE_GENERATOR in CMake
            # 3.15+.
            if not cmake_generator or cmake_generator == "Ninja":
                try:
                    import ninja  # noqa: F401

                    ninja_executable_path = Path(ninja.BIN_DIR) / "ninja"
                    cmake_args += [
                        "-GNinja",
                        f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}",
                    ]
                except ImportError:
                    pass

        else:

            # Single config generators are handled "normally"
            single_config = any(x in cmake_generator for x in {"NMake", "Ninja"})

            # CMake allows an arch-in-generator style for backward compatibility
            contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"})

            # Specify the arch if using MSVC generator, but only if it doesn't
            # contain a backward-compatibility arch spec already in the
            # generator name.
            if not single_config and not contains_arch:
                cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]]

            # Multi-config generators have a different way to specify configs
            if not single_config:
                cmake_args += [
                    f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"
                ]
                build_args += ["--config", cfg]

        if sys.platform.startswith("darwin"):
            # Cross-compile support for macOS - respect ARCHFLAGS if set
            archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
            if archs:
                cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]

        # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level
        # across all generators.
        if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
            # self.parallel is a Python 3 only way to set parallel jobs by hand
            # using -j in the build_ext call, not supported by pip or PyPA-build.
            if hasattr(self, "parallel") and self.parallel:
                # CMake 3.12+ only.
                build_args += [f"-j{self.parallel}"]

        build_temp = Path(self.build_temp) / ext.name
        if not build_temp.exists():
            build_temp.mkdir(parents=True)

        subprocess.run(
            ["cmake", ext.sourcedir] + cmake_args, cwd=build_temp, check=True
        )
        subprocess.run(
            ["cmake", "--build", "."] + build_args, cwd=build_temp, check=True
        )


setup(
    # Currently, build_ext only provides an optional "highest supported C++
    # level" feature, but in the future it may provide more features.
    ext_modules=[CMakeExtension("mymodule")],
    cmdclass={"build_ext": CMakeBuild},
    zip_safe=False,
)

• src/main.cpp: The content is from xtensor-python github

#include <numeric>
#include <pybind11/pybind11.h>
#include <xtensor/xmath.hpp>
#define FORCE_IMPORT_ARRAY
#include <xtensor-python/pyarray.hpp>

double sum_of_sines(xt::pyarray<double> &m) {
  auto sines = xt::sin(m); // sines does not actually hold values.
  return std::accumulate(sines.begin(), sines.end(), 0.0);
}

PYBIND11_MODULE(mymodule, m) {
  xt::import_numpy();
  m.doc() = "Test module for xtensor python bindings";
  m.def("sum_of_sines", sum_of_sines, "Sum the sines of the input values");
}

Installation:

• Create venv, activate venv, and update pip:

python3 -m venv venv && source venv/bin/activate && python3 -m pip install --upgrade pip

• Install from Github repository directly:

python3 -m pip install "mymodule @ git+ssh://[email protected]/lionlai1989/xtensor_pybind11_cmake.git"

• (Optional) Git clone repository and pip install:

git clone --recursive https://github.com/lionlai1989/xtensor_pybind11_cmake.git
python3 -m pip install .

Verify installation:

>>> import numpy as np
>>> import mymodule
>>> v = np.arange(15).reshape(3, 5)
>>> mymodule.sum_of_sines(v)
1.2853996391883833

Everything works perfectly so far.

Enable optimization and xsimd

Now I want to enable optimization. Follow the xtensor documentation

Add xtensor::optimize and xtensor::use_xsimd

• Uncomment Line:44 in CMakeLists.txt:

target_link_libraries(mymodule PUBLIC xtensor pybind11::module xtensor-python Python::NumPy xtensor::optimize xtensor::use_xsimd)

• Run pip install . again, and the error shows cannot find target xtensor::optimize:

Building wheels for collected packages: mymodule
  Building wheel for mymodule (pyproject.toml) ... error
  error: subprocess-exited-with-error
  
  × Building wheel for mymodule (pyproject.toml) did not run successfully.
  │ exit code: 1
  ╰─> [86 lines of output]
      running bdist_wheel
      running build
      running build_ext
      -- pybind11 v2.11.0 dev1
      -- Found PythonInterp: /home/lai/xtensor_pybind11_cmake/venv/bin/python3 (found suitable version "3.8.10", minimum required is "3.6")
      -- xtl v0.7.5
      -- Building xtensor v0.24.6
      -- Found xtl v0.7.5
      -- xsimd v10.0.0
      -- xtensor-python v0.26.1
      -- Found xtensor v0.24.6
      -- Found pybind11 v
      -- Found PythonInterp: /home/lai/xtensor_pybind11_cmake/venv/bin/python3 (found version "3.8.10")
      -- Found NumPy: version "1.24.2" /tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/numpy/core/include
      -- Found numpy: /tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/numpy/core/include
      -- Found Python: /home/lai/xtensor_pybind11_cmake/venv/bin/python3 (found version "3.8.10") found components: Interpreter Development NumPy Development.Module Development.Embed
      -- Configuring done (0.4s)
      CMake Error at CMakeLists.txt:43 (target_link_libraries):
        Target "mymodule" links to:
      
          xtensor::optimize
      
        but the target was not found.  Possible reasons include:
      
          * There is a typo in the target name.
          * A find_package call is missing for an IMPORTED target.
          * An ALIAS target is missing.
      
      
      
      -- Generating done (0.0s)
      CMake Generate step failed.  Build files cannot be regenerated correctly.
      Traceback (most recent call last):
        File "/home/lai/xtensor_pybind11_cmake/venv/lib/python3.8/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 353, in <module>
          main()
        File "/home/lai/xtensor_pybind11_cmake/venv/lib/python3.8/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 335, in main
          json_out['return_val'] = hook(**hook_input['kwargs'])
        File "/home/lai/xtensor_pybind11_cmake/venv/lib/python3.8/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 251, in build_wheel
          return _build_backend().build_wheel(wheel_directory, config_settings,
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/build_meta.py", line 404, in build_wheel
          return self._build_with_temp_dir(['bdist_wheel'], '.whl',
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/build_meta.py", line 389, in _build_with_temp_dir
          self.run_setup()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/build_meta.py", line 334, in run_setup
          exec(code, locals())
        File "<string>", line 127, in <module>
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/__init__.py", line 87, in setup
          return distutils.core.setup(**attrs)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/core.py", line 185, in setup
          return run_commands(dist)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/core.py", line 201, in run_commands
          dist.run_commands()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/dist.py", line 973, in run_commands
          self.run_command(cmd)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/dist.py", line 1217, in run_command
          super().run_command(command)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/dist.py", line 992, in run_command
          cmd_obj.run()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/wheel/bdist_wheel.py", line 343, in run
          self.run_command("build")
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/cmd.py", line 319, in run_command
          self.distribution.run_command(command)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/dist.py", line 1217, in run_command
          super().run_command(command)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/dist.py", line 992, in run_command
          cmd_obj.run()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/command/build.py", line 132, in run
          self.run_command(cmd_name)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/cmd.py", line 319, in run_command
          self.distribution.run_command(command)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/dist.py", line 1217, in run_command
          super().run_command(command)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/dist.py", line 992, in run_command
          cmd_obj.run()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/command/build_ext.py", line 84, in run
          _build_ext.run(self)
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py", line 346, in run
          self.build_extensions()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py", line 466, in build_extensions
          self._build_extensions_serial()
        File "/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py", line 492, in _build_extensions_serial
          self.build_extension(ext)
        File "<string>", line 119, in build_extension
        File "/usr/lib/python3.8/subprocess.py", line 516, in run
          raise CalledProcessError(retcode, process.args,
      subprocess.CalledProcessError: Command '['cmake', '/home/lai/xtensor_pybind11_cmake', '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=/home/lai/xtensor_pybind11_cmake/build/lib.linux-x86_64-cpython-38/', '-DPYTHON_EXECUTABLE=/home/lai/xtensor_pybind11_cmake/venv/bin/python3', '-DCMAKE_BUILD_TYPE=Release', '-DEXAMPLE_VERSION_INFO=0.0.1', '-GNinja', '-DCMAKE_MAKE_PROGRAM:FILEPATH=/tmp/pip-build-env-d762ck7q/overlay/lib/python3.8/site-packages/ninja/data/bin/ninja']' returned non-zero exit status 1.
      [end of output]
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
  ERROR: Failed building wheel for mymodule
Failed to build mymodule
ERROR: Could not build wheels for mymodule, which is required to install pyproject.toml-based projects

Use find_package in CMakeLists.txt:

My previous experience tells me that using find_package could solve this problem. Thus, I tried:

• Install xtl, xtensor and xsimd in the computer under/tmp:

(cd extern/xtl && cmake -D CMAKE_INSTALL_PREFIX=/tmp/xtl-install && make install)
(cd extern/xtensor && cmake -D CMAKE_INSTALL_PREFIX=/tmp/xtensor-install -DCMAKE_PREFIX_PATH=/tmp/xtl-install && make install)
(cd extern/xsimd && cmake -D CMAKE_INSTALL_PREFIX=/tmp/xsimd-install && make install)

• Delete add_subdirectory and add find_package in CMakeLists.txt:

list(APPEND CMAKE_PREFIX_PATH "/tmp/xtl-install")
list(APPEND CMAKE_PREFIX_PATH "/tmp/xtensor-install")
list(APPEND CMAKE_PREFIX_PATH "/tmp/xsimd-install")
find_package(xtl REQUIRED)
find_package(xtensor REQUIRED)
find_package(xsimd REQUIRED)

• Run pip install . again with xtensor::optimize xtensor::use_xsimd. And everything works. However, the problem is that xtl, xtensor, xsimd packages cannot be installed before users running pip install command.

Try FetchContent

I wan thinking if add_subdirectory doesn't work, maybe FetchContent will work. Unfortunately, it's not the case. The same error message still exists. You can uncomment the code block of FetchContent in CMakeLists.txt to test it out.

My questions are:

1. How can I make the target mymodule link to xtensor::optimize and xtensor::use_xsimd when using add_subdirectory or FetchContent?
2. If 1.) is not possible, is there a way to automate the installation of xtensor, xtl, and xsimd in CMake and pip installation process so that find_package can be used in CMakeLists.txt. Subsequently, anyone can install this package with a single and simple pip install.

System Information:

• Ubuntu 20.04
• Python 3.8.10
• All submodules are at master branch.
• All other system build requirements are fixed by pyproject.toml.

(I also posted the same question on StackOverflow)