Merge pull request #460 from bluescarni/pr/force_load

Implement approach to manually inject heyoka's symbols in the JIT

CMakeLists.txt

@@ -11,7 +11,7 @@ if(NOT CMAKE_BUILD_TYPE)
 	FORCE)
 endif()
 
-project(heyoka VERSION 6.0.0 LANGUAGES CXX C)
+project(heyoka VERSION 6.1.0 LANGUAGES CXX C)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/yacma")
 
@@ -194,6 +194,7 @@ set(HEYOKA_SRC_FILES
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/debug.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/aligned_buffer.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/type_traits.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/get_dl_path.cpp"
     # NOTE: this will be an empty file in case we are not
     # building with support for real.
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/real_helpers.cpp"
@@ -336,7 +337,7 @@ if(HEYOKA_WITH_SLEEF)
 endif()
 
 # Setup the heyoka ABI version number.
-set(HEYOKA_ABI_VERSION 30)
+set(HEYOKA_ABI_VERSION 31)
 
 if(HEYOKA_BUILD_STATIC_LIBRARY)
     # Setup of the heyoka static library.

doc/changelog.rst

@@ -1,12 +1,15 @@
 Changelog
 =========
 
-6.0.1 (unreleased)
+6.1.0 (unreleased)
 ------------------
 
 Fix
 ~~~
 
+- Fix symbols not found by the JIT runtime when heyoka
+  is ``dlopen()``-ed with ``RTLD_LOCAL``
+  (`#460 <https://github.com/bluescarni/heyoka/pull/460>`__).
 - Workaround for a clang 17 issue that would result in
   runtime exceptions during (de)serialisation
   (`#458 <https://github.com/bluescarni/heyoka/pull/458>`__).

include/heyoka/detail/get_dl_path.hpp

@@ -0,0 +1,27 @@
+// Copyright 2020, 2021, 2022, 2023, 2024 Francesco Biscani ([email protected]), Dario Izzo ([email protected])
+//
+// This file is part of the heyoka library.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef HEYOKA_DETAIL_GET_DL_PATH_HPP
+#define HEYOKA_DETAIL_GET_DL_PATH_HPP
+
+#include <string>
+
+#include <heyoka/config.hpp>
+
+HEYOKA_BEGIN_NAMESPACE
+
+namespace detail
+{
+
+const std::string &get_dl_path();
+
+}
+
+HEYOKA_END_NAMESPACE
+
+#endif

src/detail/get_dl_path.cpp

@@ -0,0 +1,96 @@
+// Copyright 2020, 2021, 2022, 2023, 2024 Francesco Biscani ([email protected]), Dario Izzo ([email protected])
+//
+// This file is part of the heyoka library.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <exception>
+#include <filesystem>
+#include <iostream>
+#include <string>
+
+#if __has_include(<dlfcn.h>)
+
+#define HEYOKA_DETAIL_GET_DL_PATH_DLFCN
+
+#include <dlfcn.h>
+
+#endif
+
+#include <heyoka/config.hpp>
+#include <heyoka/detail/get_dl_path.hpp>
+
+HEYOKA_BEGIN_NAMESPACE
+
+namespace detail
+{
+
+namespace
+{
+
+#if defined(HEYOKA_DETAIL_GET_DL_PATH_DLFCN)
+
+// NOTE: need a dummy variable to take the address of.
+const int dummy = 42;
+
+// Implementation of make_dl_path() based on dladdr(), from the dlfcn.h header.
+std::string make_dl_path_dlfcn()
+{
+    // Invoke dladdr().
+    ::Dl_info dl_info;
+    const auto ret = ::dladdr(static_cast<const void *>(&dummy), &dl_info);
+
+    // NOTE: in case of any failure, we will fall through the
+    // "return {};" statement and produce an empty string.
+    if (ret != 0 && dl_info.dli_fname != nullptr) {
+        try {
+            return std::filesystem::canonical(std::filesystem::path(dl_info.dli_fname)).string();
+            // LCOV_EXCL_START
+        } catch (const std::exception &e) {
+            std::cerr << "WARNING - exception raised while trying to determine the path of the heyoka library: "
+                      << e.what() << std::endl;
+        } catch (...) {
+            std::cerr << "WARNING - exception raised while trying to determine the path of the heyoka library"
+                      << std::endl;
+        }
+    }
+
+    return {};
+    // LCOV_EXCL_STOP
+}
+
+#endif
+
+// Factory function for dl_path.
+std::string make_dl_path()
+{
+#if defined(HEYOKA_DETAIL_GET_DL_PATH_DLFCN)
+
+    return make_dl_path_dlfcn();
+
+#else
+
+    return {};
+
+#endif
+}
+
+// The path to the heyoka shared library.
+const std::string dl_path = make_dl_path();
+
+} // namespace
+
+// This function is meant to return the full canonicalised path to the heyoka shared library.
+//
+// If, for any reason, the path cannot be determined, an empty
+// string will be returned instead.
+const std::string &get_dl_path()
+{
+    return detail::dl_path;
+}
+
+} // namespace detail
+
+HEYOKA_END_NAMESPACE

src/llvm_state.cpp

@@ -102,6 +102,7 @@
 
 #endif
 
+#include <heyoka/detail/get_dl_path.hpp>
 #include <heyoka/detail/llvm_func_create.hpp>
 #include <heyoka/detail/llvm_fwd.hpp>
 #include <heyoka/detail/llvm_helpers.hpp>
@@ -720,6 +721,31 @@ struct llvm_state::jit {
         // LCOV_EXCL_STOP
         m_lljit->getMainJITDylib().addGenerator(std::move(*dlsg));
 
+        // NOTE: we also want to manually inject the symbols from the heyoka shared library
+        // into the JIT runtime.
+        //
+        // The reason for this is that if heyoka is loaded with dlopen() and the RTLD_LOCAL flag,
+        // then the JIT runtime will not be able to resolve symbols defined either in heyoka or
+        // in its dependencies. This will happen for instance in heyoka.py.
+        //
+        // With the following contraption, as far as I have understood, we are manually injecting all the
+        // symbols from heyoka (and, transitively, its dependencies) into the JIT runtime, and the symbols
+        // are thus made available to JIT code despite the use of RTLD_LOCAL.
+        //
+        // This approach was suggested by lhames on the LLVM discord.
+        if (const auto &dl_path = detail::get_dl_path(); !dl_path.empty()) {
+            auto new_dlsg = llvm::orc::DynamicLibrarySearchGenerator::Load(dl_path.c_str(),
+                                                                           m_lljit->getDataLayout().getGlobalPrefix());
+            if (new_dlsg) [[likely]] {
+                m_lljit->getMainJITDylib().addGenerator(std::move(*new_dlsg));
+            } else {
+                // LCOV_EXCL_START
+                throw std::invalid_argument(
+                    "Could not create the dynamic library search generator for the heyoka library");
+                // LCOV_EXCL_STOP
+            }
+        }
+
         // Keep a target machine around to fetch various
         // properties of the host CPU.
         auto tm = jtmb.createTargetMachine();
@@ -735,7 +761,7 @@ struct llvm_state::jit {
 
         // NOTE: by default, errors in the execution session are printed
         // to screen. A custom error reported can be specified, ideally
-        // we would like th throw here but I am not sure whether throwing
+        // we would like to throw here but I am not sure whether throwing
         // here would disrupt LLVM's cleanup actions?
         // https://llvm.org/doxygen/classllvm_1_1orc_1_1ExecutionSession.html
 
@@ -1802,6 +1828,30 @@ multi_jit::multi_jit(unsigned n_modules, unsigned opt_level, code_model c_model,
     // LCOV_EXCL_STOP
     m_lljit->getMainJITDylib().addGenerator(std::move(*dlsg));
 
+    // NOTE: we also want to manually inject the symbols from the heyoka shared library
+    // into the JIT runtime.
+    //
+    // The reason for this is that if heyoka is loaded with dlopen() and the RTLD_LOCAL flag,
+    // then the JIT runtime will not be able to resolve symbols defined either in heyoka or
+    // in its dependencies. This will happen for instance in heyoka.py.
+    //
+    // With the following contraption, as far as I have understood, we are manually injecting all the
+    // symbols from heyoka (and, transitively, its dependencies) into the JIT runtime, and the symbols
+    // are thus made available to JIT code despite the use of RTLD_LOCAL.
+    //
+    // This approach was suggested by lhames on the LLVM discord.
+    if (const auto &dl_path = detail::get_dl_path(); !dl_path.empty()) {
+        auto new_dlsg = llvm::orc::DynamicLibrarySearchGenerator::Load(dl_path.c_str(),
+                                                                       m_lljit->getDataLayout().getGlobalPrefix());
+        if (new_dlsg) [[likely]] {
+            m_lljit->getMainJITDylib().addGenerator(std::move(*new_dlsg));
+        } else {
+            // LCOV_EXCL_START
+            throw std::invalid_argument("Could not create the dynamic library search generator for the heyoka library");
+            // LCOV_EXCL_STOP
+        }
+    }
+
     // Create the master context.
     m_ctx = std::make_unique<llvm::orc::ThreadSafeContext>(std::make_unique<llvm::LLVMContext>());
 

tools/gha_osx_x86.sh

@@ -13,7 +13,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
 bash miniconda.sh -b -p $HOME/miniconda
 conda create -y -p $deps_dir c-compiler zlib cxx-compiler libcxx cmake ninja \
     llvmdev tbb-devel tbb libboost-devel sleef xtensor xtensor-blas blas \
-    blas-devel fmt spdlog 'mppp=1.*'
+    blas-devel fmt spdlog 'mppp=1.*' 'clang<19' 'clangxx<19'
 source activate $deps_dir
 
 # Create the build dir and cd into it.

tools/gha_osx_x86_static.sh

@@ -13,7 +13,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
 bash miniconda.sh -b -p $HOME/miniconda
 conda create -y -p $deps_dir c-compiler zlib cxx-compiler libcxx cmake ninja \
     llvmdev tbb-devel tbb libboost-devel sleef xtensor xtensor-blas blas \
-    blas-devel fmt spdlog 'mppp=1.*'
+    blas-devel fmt spdlog 'mppp=1.*' 'clang<19' 'clangxx<19'
 source activate $deps_dir
 
 # Create the build dir and cd into it.