diff --git a/changelog.md b/changelog.md index 03fbe5b..672bea0 100644 --- a/changelog.md +++ b/changelog.md @@ -1,8 +1,9 @@ # Changelog -## v1.9.1 | In development +## v1.9.1 | 2024-06-17 - Fixed an issue where calling CMake with `-DPython_EXECUTABLE=` created conflicts with the embedded Python (either a loud version error, or silently passing the wrong library paths). Some IDEs would pass this flag implicitly and it would hijack the `find_package(Python)` call used internally by this recipe. Now, we specifically protect against this since there should be no traces of system Python in a project that wishes to embed it. +- Provided an alternative to `embedded_python_tools.symlink_import()`. For dev builds, it's now possible to point `PyConfig::home` to the contents of `bin/.embedded_python(-core).home` to avoid needing to copy the entire Python environment into the build tree every time the project is reconfigured. ## v1.9.0 | 2024-05-03 diff --git a/core/conanfile.py b/core/conanfile.py index d13063a..647436a 100644 --- a/core/conanfile.py +++ b/core/conanfile.py @@ -27,7 +27,8 @@ class EmbeddedPythonCore(ConanFile): default_options = { "zip_stdlib": "stored", } - exports_sources = "embedded_python_tools.py", "embedded_python-core.cmake" + exports_sources = "embedded_python_tools.py", "embedded_python*.cmake" + package_type = "shared-library" def validate(self): minimum_python = "3.11.5" @@ -277,7 +278,7 @@ def _isolate(self, prefix): def package(self): src = self.build_folder dst = pathlib.Path(self.package_folder, "embedded_python") - files.copy(self, "embedded_python-core.cmake", src, dst=self.package_folder) + files.copy(self, "embedded_python*.cmake", src, dst=self.package_folder) files.copy(self, "embedded_python_tools.py", src, dst=self.package_folder) license_folder = pathlib.Path(self.package_folder, "licenses") @@ -321,8 +322,10 @@ def package(self): def package_info(self): self.env_info.PYTHONPATH.append(self.package_folder) - self.cpp_info.set_property("cmake_build_modules", ["embedded_python-core.cmake"]) - self.cpp_info.build_modules = ["embedded_python-core.cmake"] + self.cpp_info.set_property( + "cmake_build_modules", ["embedded_python-core.cmake", "embedded_python-tools.cmake"] + ) + self.cpp_info.build_modules = ["embedded_python-core.cmake", "embedded_python-tools.cmake"] prefix = pathlib.Path(self.package_folder) / "embedded_python" self.cpp_info.includedirs = [str(prefix / "include")] if self.settings.os == "Windows": diff --git a/core/embedded_python-tools.cmake b/core/embedded_python-tools.cmake new file mode 100644 index 0000000..0438fde --- /dev/null +++ b/core/embedded_python-tools.cmake @@ -0,0 +1,15 @@ +include_guard(DIRECTORY) + +# For development, we want avoid copying all of Python's `lib` and `site-packages` into our +# build tree every time we re-configure the project. Instead, we can point `PyConfig::home` +# to the contents of this file to gain access to all the Python packages. +# For release/deployment, the entire `Python_ROOT_DIR` should be copied into the app's `bin` +# folder and `PyConfig::home` should point to that. +function(embedded_python_generate_home_file filename content) + if(DEFINED CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(filename ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${filename}) + endif() + file(GENERATE OUTPUT ${filename} CONTENT "${content}") +endfunction() + +embedded_python_generate_home_file(".embedded_python-core.home" "${Python_ROOT_DIR}") diff --git a/core/test_package/conanfile.py b/core/test_package/conanfile.py index 4872959..da5ca97 100644 --- a/core/test_package/conanfile.py +++ b/core/test_package/conanfile.py @@ -3,14 +3,14 @@ import subprocess import conan from conan import ConanFile -from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.cmake import CMake, cmake_layout # noinspection PyUnresolvedReferences class TestEmbeddedPythonCore(ConanFile): name = "test_embedded_python" settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "VirtualRunEnv" + generators = "CMakeToolchain", "CMakeDeps", "VirtualRunEnv" test_type = "explicit" def layout(self): @@ -19,18 +19,7 @@ def layout(self): def requirements(self): self.requires(self.tested_reference_str) - def generate(self): - build_type = self.settings.build_type.value - tc = CMakeToolchain(self) - tc.variables[f"CMAKE_RUNTIME_OUTPUT_DIRECTORY_{build_type.upper()}"] = "bin" - tc.generate() - def build(self): - sys.path.append(str(self._core_package_path)) - - import embedded_python_tools - - embedded_python_tools.symlink_import(self, dst="bin/python") cmake = CMake(self) cmake.configure( variables={ @@ -42,13 +31,6 @@ def build(self): ) cmake.build() - @property - def _py_exe(self): - if self.settings.os == "Windows": - return pathlib.Path(self.build_folder, "bin/python/python.exe") - else: - return pathlib.Path(self.build_folder, "bin/python/bin/python3") - @property def _core_package_path(self): if conan.__version__.startswith("2"): @@ -56,6 +38,11 @@ def _core_package_path(self): else: return pathlib.Path(self.deps_cpp_info["embedded_python-core"].rootpath) + @property + def _py_exe(self): + exe = "python.exe" if sys.platform == "win32" else "python3" + return self._core_package_path / "embedded_python" / exe + def _test_stdlib(self): """Ensure that Python runs and built the optional stdlib modules""" self.run(f'{self._py_exe} -c "import sys; print(sys.version);"') @@ -78,7 +65,7 @@ def _test_libpython_path(self): def _test_embed(self): """Ensure that everything is available to compile and link to the embedded Python""" - self.run(pathlib.Path("bin", "test_package"), env="conanrun") + self.run(pathlib.Path(self.cpp.build.bindir, "test_package").absolute(), env="conanrun") def _test_licenses(self): """Ensure that the license file is included""" diff --git a/core/test_package/src/main.cpp b/core/test_package/src/main.cpp index 9bac7c2..911012e 100644 --- a/core/test_package/src/main.cpp +++ b/core/test_package/src/main.cpp @@ -1,13 +1,29 @@ #include #include +#include #include +std::string find_python_home(std::filesystem::path bin) { + const auto local_home = bin / "python"; + if (std::filesystem::exists(local_home)) { + return local_home.string(); + } + + auto home_file = bin / ".embedded_python.home"; + if (!std::filesystem::exists(home_file)) { + home_file = bin / ".embedded_python-core.home"; + } + auto stream = std::ifstream(home_file); + return std::string(std::istreambuf_iterator(stream), + std::istreambuf_iterator()); +} + int main(int argc, const char* argv[]) { auto config = PyConfig{}; PyConfig_InitIsolatedConfig(&config); const auto bin = std::filesystem::path(argv[0]).parent_path(); - const auto python_home = (bin / "python").string(); + const auto python_home = find_python_home(bin); if (auto status = PyConfig_SetBytesString(&config, &config.home, python_home.c_str()); PyStatus_Exception(status)) { PyConfig_Clear(&config); diff --git a/embedded_python.cmake b/embedded_python.cmake index 33d2104..03ad8af 100644 --- a/embedded_python.cmake +++ b/embedded_python.cmake @@ -1,3 +1,5 @@ +include_guard(DIRECTORY) + # There is one important thing we want to achieve with the `embedded_python`/`embedded_python-core` # split: we want to avoid recompiling the world when only the Python environment packages change # but the version/headers/libs stay the same. To do that we must ensure that everything is built @@ -9,9 +11,13 @@ # library. On top of that, `embedded_python.cmake` adds `EmbeddedPython_EXECUTABLE` which is aware # of the full environment with `pip` packages. Note that we do no provide any include or lib dirs # since those are already provided by `core`. - +set(EmbeddedPython_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/embedded_python" CACHE STRING "" FORCE) if(WIN32) - set(EmbeddedPython_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/embedded_python/python.exe" CACHE STRING "" FORCE) + set(EmbeddedPython_EXECUTABLE "${EmbeddedPython_ROOT_DIR}/python.exe" CACHE STRING "" FORCE) else() - set(EmbeddedPython_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/embedded_python/python3" CACHE STRING "" FORCE) + set(EmbeddedPython_EXECUTABLE "${EmbeddedPython_ROOT_DIR}/python3" CACHE STRING "" FORCE) endif() + +# See docstring of `embedded_python_generate_home_file()`. It's up to the user to pick if they +# want to point the `-core` package (no `pip` package) or the full embedded environment. +embedded_python_generate_home_file(".embedded_python.home" "${EmbeddedPython_ROOT_DIR}") diff --git a/test_package/conanfile.py b/test_package/conanfile.py index 82d967c..4e0e19f 100644 --- a/test_package/conanfile.py +++ b/test_package/conanfile.py @@ -3,7 +3,7 @@ import subprocess import conan from conan import ConanFile -from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.cmake import CMake, cmake_layout project_root = pathlib.Path(__file__).parent @@ -17,12 +17,13 @@ def _read_env(name): class TestEmbeddedPython(ConanFile): name = "test_embedded_python" settings = "os", "compiler", "build_type", "arch" - generators = "CMakeDeps", "VirtualRunEnv" + generators = "CMakeToolchain", "CMakeDeps", "VirtualRunEnv" options = {"env": [None, "ANY"]} default_options = { "env": None, "embedded_python-core/*:version": "3.11.5", } + package_type = "shared-library" @property def _core_package_path(self): @@ -38,6 +39,11 @@ def _package_path(self): else: return pathlib.Path(self.deps_cpp_info["embedded_python"].rootpath) + @property + def _py_exe(self): + exe = "python.exe" if sys.platform == "win32" else "python3" + return self._package_path / "embedded_python" / exe + def layout(self): cmake_layout(self) @@ -48,19 +54,7 @@ def configure(self): if self.options.env: self.options["embedded_python"].packages = _read_env(self.options.env) - def generate(self): - build_type = self.settings.build_type.value - tc = CMakeToolchain(self) - tc.variables[f"CMAKE_RUNTIME_OUTPUT_DIRECTORY_{build_type.upper()}"] = "bin" - tc.generate() - def build(self): - sys.path.append(str(self._package_path)) - - import embedded_python_tools - - embedded_python_tools.symlink_import(self, dst="bin/python") - cmake = CMake(self) cmake.configure( variables={ @@ -75,22 +69,17 @@ def build(self): def _test_env(self): """Ensure that Python runs and finds the installed environment""" - if self.settings.os == "Windows": - python_exe = str(pathlib.Path("./bin/python/python").resolve()) - else: - python_exe = str(pathlib.Path("./bin/python/bin/python3").resolve()) - - self.run(f'{python_exe} -c "import sys; print(sys.version);"') - + self.run(f'{self._py_exe} -c "import sys; print(sys.version);"') name = str(self.options.env) if self.options.env else "baseline" - self.run(f"{python_exe} {project_root / name / 'test.py'}", env="conanrun") + self.run(f"{self._py_exe} {project_root / name / 'test.py'}", env="conanrun") def _test_libpython_path(self): if self.settings.os != "Macos": return - python_exe = str(pathlib.Path("./bin/python/bin/python3").resolve()) - p = subprocess.run(["otool", "-L", python_exe], check=True, text=True, capture_output=True) + p = subprocess.run( + ["otool", "-L", self._py_exe], check=True, text=True, capture_output=True + ) lines = str(p.stdout).strip().split("\n")[1:] libraries = [line.split()[0] for line in lines] candidates = [lib for lib in libraries if "libpython" in lib] @@ -101,7 +90,7 @@ def _test_libpython_path(self): def _test_embed(self): """Ensure that everything is available to compile and link to the embedded Python""" - self.run(pathlib.Path("bin", "test_package"), env="conanrun") + self.run(pathlib.Path(self.cpp.build.bindir, "test_package").absolute(), env="conanrun") def _test_licenses(self): """Ensure that the licenses have been gathered"""