Merge pull request #340 from bluescarni/pr/mem_cache

In-memory cache for llvm_state

CMakeLists.txt

@@ -278,6 +278,7 @@ set(HEYOKA_SRC_FILES
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/math_wrappers.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/logging_impl.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/src/step_callback.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/llvm_state_mem_cache.cpp"
     # NOTE: sleef.cpp needs to be compiled even if we are not
     # building with sleef support on.
     "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/sleef.cpp"

config.hpp.in

@@ -88,4 +88,17 @@
     }
 // clang-format on
 
+// C++20 constinit.
+// NOTE: this seems to be buggy currently on MSVC:
+// https://github.com/bluescarni/mppp/issues/291
+#if defined(__cpp_constinit) && (!defined(_MSC_VER) || defined(__clang__))
+
+#define HEYOKA_CONSTINIT constinit
+
+#else
+
+#define HEYOKA_CONSTINIT
+
+#endif
+
 #endif

doc/changelog.rst

@@ -7,13 +7,20 @@ Changelog
 New
 ~~~
 
+- Implement an in-memory cache for ``llvm_state``. The cache is used
+  to avoid re-optimising and re-compiling LLVM code which has
+  already been optimised and compiled during the program execution
+  (`#340 <https://github.com/bluescarni/heyoka/pull/340>`__).
 - It is now possible to get the LLVM bitcode of
   an ``llvm_state``
   (`#339 <https://github.com/bluescarni/heyoka/pull/339>`__).
 
 Changes
 ~~~~~~~
 
+- The optimisation level for an ``llvm_state`` is now clamped
+  within the ``[0, 3]`` range
+  (`#340 <https://github.com/bluescarni/heyoka/pull/340>`__).
 - The LLVM bitcode is now used internally (instead of the textual
   representation of the IR) when copying and serialising
   an ``llvm_state``
@@ -25,6 +32,8 @@ Changes
 Fix
 ~~~
 
+- Fix compilation in C++20 mode
+  (`#340 <https://github.com/bluescarni/heyoka/pull/340>`__).
 - Fix the object file of an ``llvm_state`` not being
   preserved during copy and deserialisation
   (`#339 <https://github.com/bluescarni/heyoka/pull/339>`__).

include/heyoka/detail/llvm_func_create.hpp

@@ -0,0 +1,55 @@
+// Copyright 2020, 2021, 2022, 2023 Francesco Biscani ([email protected]), Dario Izzo ([email protected])
+//
+// This file is part of the heyoka library.
+//
+// This Source Code Form is subject to the terms of the Mozilla
+// Public License v. 2.0. If a copy of the MPL was not distributed
+// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef HEYOKA_DETAIL_LLVM_FUNC_CREATE_HPP
+#define HEYOKA_DETAIL_LLVM_FUNC_CREATE_HPP
+
+#include <cassert>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+#include <fmt/core.h>
+
+#include <llvm/IR/DerivedTypes.h>
+#include <llvm/IR/Function.h>
+
+#include <heyoka/config.hpp>
+
+HEYOKA_BEGIN_NAMESPACE
+
+namespace detail
+{
+
+// Helper to create an LLVM function.
+// NOTE: the purpose of this helper is to check that the function was created with
+// the requested name: LLVM will silently change the name of the created function
+// if it already exists in a module, and in some cases we want to prevent
+// this from happening.
+template <typename... Args>
+llvm::Function *llvm_func_create(llvm::FunctionType *tp, llvm::Function::LinkageTypes linkage, const std::string &name,
+                                 Args &&...args)
+{
+    llvm::Function *ret = llvm::Function::Create(tp, linkage, name, std::forward<Args>(args)...);
+    assert(ret != nullptr);
+
+    if (ret->getName() != name) {
+        // Remove function before throwing.
+        ret->eraseFromParent();
+
+        throw std::invalid_argument(fmt::format("Unable to create an LLVM function with name '{}'", name));
+    }
+
+    return ret;
+}
+
+} // namespace detail
+
+HEYOKA_END_NAMESPACE
+
+#endif

include/heyoka/llvm_state.hpp

@@ -155,6 +155,10 @@ class HEYOKA_DLL_PUBLIC llvm_state
     HEYOKA_DLL_LOCAL void check_uncompiled(const char *) const;
     HEYOKA_DLL_LOCAL void check_compiled(const char *) const;
 
+    // Helper to clamp the optimisation level to
+    // the [0, 3] range.
+    static unsigned clamp_opt_level(unsigned);
+
     // Implementation details for the variadic constructor.
     template <typename... KwArgs>
     static auto kw_args_ctor_impl(KwArgs &&...kw_args)
@@ -183,6 +187,7 @@ class HEYOKA_DLL_PUBLIC llvm_state
                     return 3;
                 }
             }();
+            opt_level = clamp_opt_level(opt_level);
 
             // Fast math flag (defaults to false).
             auto fmath = [&p]() -> bool {
@@ -211,6 +216,10 @@ class HEYOKA_DLL_PUBLIC llvm_state
     // end of a constructor.
     HEYOKA_DLL_LOCAL void ctor_setup_math_flags();
 
+    // Low-level implementation details for compilation.
+    HEYOKA_DLL_LOCAL void compile_impl();
+    HEYOKA_DLL_LOCAL void add_obj_trigger();
+
     // Meta-programming for the kwargs ctor. Enabled if:
     // - there is at least 1 argument (i.e., cannot act as a def ctor),
     // - if there is only 1 argument, it cannot be of type llvm_state
@@ -237,15 +246,15 @@ class HEYOKA_DLL_PUBLIC llvm_state
     llvm::Module &module();
     ir_builder &builder();
     llvm::LLVMContext &context();
-    unsigned &opt_level();
 
     [[nodiscard]] const std::string &module_name() const;
     [[nodiscard]] const llvm::Module &module() const;
     [[nodiscard]] const ir_builder &builder() const;
     [[nodiscard]] const llvm::LLVMContext &context() const;
-    [[nodiscard]] const unsigned &opt_level() const;
     [[nodiscard]] bool fast_math() const;
     [[nodiscard]] bool force_avx512() const;
+    [[nodiscard]] unsigned get_opt_level() const;
+    void set_opt_level(unsigned);
 
     [[nodiscard]] std::string get_ir() const;
     [[nodiscard]] std::string get_bc() const;
@@ -258,21 +267,41 @@ class HEYOKA_DLL_PUBLIC llvm_state
     void optimise();
 
     [[nodiscard]] bool is_compiled() const;
-    [[nodiscard]] bool has_object_code() const;
 
     void compile();
 
     std::uintptr_t jit_lookup(const std::string &);
 
     [[nodiscard]] llvm_state make_similar() const;
+
+    // Cache management.
+    static std::size_t get_memcache_size();
+    static std::size_t get_memcache_limit();
+    static void set_memcache_limit(std::size_t);
+    static void clear_memcache();
 };
 
+namespace detail
+{
+
+// The value contained in the in-memory cache.
+struct llvm_mc_value {
+    std::string opt_bc, opt_ir, obj;
+};
+
+// Cache lookup and insertion.
+std::optional<llvm_mc_value> llvm_state_mem_cache_lookup(const std::string &, unsigned);
+void llvm_state_mem_cache_try_insert(std::string, unsigned, llvm_mc_value);
+
+} // namespace detail
+
 HEYOKA_END_NAMESPACE
 
 // Archive version changelog:
 // - version 1: got rid of the inline_functions setting;
 // - version 2: added the force_avx512 setting;
-// - version 3: added the bitcode snapshot.
+// - version 3: added the bitcode snapshot, simplified
+//   compilation logic.
 BOOST_CLASS_VERSION(heyoka::llvm_state, 3)
 
 #endif

src/detail/event_detection.cpp

@@ -27,6 +27,7 @@
 #include <boost/math/policies/policy.hpp>
 #include <boost/math/tools/toms748_solve.hpp>
 #include <boost/numeric/conversion/cast.hpp>
+#include <boost/safe_numerics/safe_integer.hpp>
 
 #if defined(HEYOKA_HAVE_REAL128)
 
@@ -58,6 +59,7 @@
 
 #include <heyoka/detail/event_detection.hpp>
 #include <heyoka/detail/fwd_decl.hpp>
+#include <heyoka/detail/llvm_func_create.hpp>
 #include <heyoka/detail/llvm_helpers.hpp>
 #include <heyoka/detail/logging_impl.hpp>
 #include <heyoka/detail/num_utils.hpp>
@@ -259,23 +261,15 @@ std::tuple<T, int> bracketed_root_find(const T *poly, std::uint32_t order, T lb,
 
     // NOTE: iter limit will be derived from the number of binary digits
     // in the significand.
-    const auto iter_limit = [&]() {
+    const boost::uintmax_t iter_limit = [&]() {
 #if defined(HEYOKA_HAVE_REAL)
         if constexpr (std::is_same_v<T, mppp::real>) {
             // NOTE: we use lb here, but any of lb, ub or the poly
             // coefficients should be guaranteed to have at least the
             // working precision of the root finding scheme.
             // NOTE: since we use bisection for mppp::real, we need to allow
             // for more iterations than the number of digits.
-            const auto ret = boost::numeric_cast<boost::uintmax_t>(lb.get_prec());
-
-            // LCOV_EXCL_START
-            if (ret > std::numeric_limits<boost::uintmax_t>::max() / 2u) {
-                throw std::overflow_error("Overflow condition detected in bracketed_root_find()");
-            }
-            // LCOV_EXCL_STOP
-
-            return ret * 2u;
+            return boost::safe_numerics::safe<boost::uintmax_t>(lb.get_prec()) * 2;
         } else {
 #endif
             return boost::numeric_cast<boost::uintmax_t>(std::numeric_limits<T>::digits);
@@ -401,12 +395,7 @@ llvm::Function *add_poly_translator_1(llvm_state &s, llvm::Type *fp_t, std::uint
     auto *ft = llvm::FunctionType::get(builder.getVoidTy(), fargs, false);
     assert(ft != nullptr); // LCOV_EXCL_LINE
     // Now create the function.
-    auto *f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, "poly_translate_1", &s.module());
-    // LCOV_EXCL_START
-    if (f == nullptr) {
-        throw std::invalid_argument("Unable to create a function for polynomial translation");
-    }
-    // LCOV_EXCL_STOP
+    auto *f = llvm_func_create(ft, llvm::Function::ExternalLinkage, "poly_translate_1", &s.module());
 
     // Set the names/attributes of the function arguments.
     auto *out_ptr = f->args().begin();
@@ -582,12 +571,7 @@ llvm::Function *llvm_add_poly_rtscc(llvm_state &s, llvm::Type *fp_t, std::uint32
     auto *ft = llvm::FunctionType::get(builder.getVoidTy(), fargs, false);
     assert(ft != nullptr); // LCOV_EXCL_LINE
     // Now create the function.
-    auto *f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, "poly_rtscc", &md);
-    // LCOV_EXCL_START
-    if (f == nullptr) {
-        throw std::invalid_argument("Unable to create an rtscc function");
-    }
-    // LCOV_EXCL_STOP
+    auto *f = llvm_func_create(ft, llvm::Function::ExternalLinkage, "poly_rtscc", &md);
 
     // Set the names/attributes of the function arguments.
     // NOTE: out_ptr1/2 are used both in read and write mode,
@@ -694,12 +678,7 @@ llvm::Function *llvm_add_fex_check(llvm_state &s, llvm::Type *fp_t, std::uint32_
     auto *ft = llvm::FunctionType::get(builder.getVoidTy(), fargs, false);
     assert(ft != nullptr); // LCOV_EXCL_LINE
     // Now create the function.
-    auto *f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, "fex_check", &md);
-    // LCOV_EXCL_START
-    if (f == nullptr) {
-        throw std::invalid_argument("Unable to create an fex_check function");
-    }
-    // LCOV_EXCL_STOP
+    auto *f = llvm_func_create(ft, llvm::Function::ExternalLinkage, "fex_check", &md);
 
     // Set the names/attributes of the function arguments.
     auto *cf_ptr = f->args().begin();

src/detail/llvm_helpers.cpp

@@ -73,6 +73,7 @@
 
 #endif
 
+#include <heyoka/detail/llvm_func_create.hpp>
 #include <heyoka/detail/llvm_fwd.hpp>
 #include <heyoka/detail/llvm_helpers.hpp>
 #include <heyoka/detail/llvm_vector_type.hpp>
@@ -209,7 +210,7 @@ llvm::Type *to_llvm_type_impl(llvm::LLVMContext &c, const std::type_info &tp, bo
 {
     const auto it = type_map.find(tp);
 
-    const auto *err_msg = "Unable to associate the C++ type '{}' to an LLVM type";
+    constexpr auto *err_msg = "Unable to associate the C++ type '{}' to an LLVM type";
 
     if (it == type_map.end()) {
         // LCOV_EXCL_START
@@ -790,11 +791,7 @@ llvm::CallInst *llvm_invoke_external(llvm_state &s, const std::string &name, llv
             arg_types.push_back(a->getType());
         }
         auto *ft = llvm::FunctionType::get(ret_type, arg_types, false);
-        callee_f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, name, &s.module());
-        if (callee_f == nullptr) {
-            throw std::invalid_argument(
-                fmt::format("Unable to create the prototype for the external function '{}'", name));
-        }
+        callee_f = llvm_func_create(ft, llvm::Function::ExternalLinkage, name, &s.module());
 
         // Add the function attributes.
         for (const auto &att : attrs) {
@@ -3194,8 +3191,7 @@ void llvm_add_inv_kep_E_wrapper(llvm_state &s, llvm::Type *scal_t, std::uint32_t
     // The return type is void.
     auto *ft = llvm::FunctionType::get(builder.getVoidTy(), fargs, false);
     // Create the function
-    auto *f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, name, &md);
-    assert(f != nullptr); // LCOV_EXCL_LINE
+    auto *f = llvm_func_create(ft, llvm::Function::ExternalLinkage, name, &md);
 
     // Fetch the current insertion block.
     auto *orig_bb = builder.GetInsertBlock();

src/expression_cfunc.cpp

@@ -63,6 +63,7 @@
 
 #include <heyoka/detail/cm_utils.hpp>
 #include <heyoka/detail/func_cache.hpp>
+#include <heyoka/detail/llvm_func_create.hpp>
 #include <heyoka/detail/llvm_fwd.hpp>
 #include <heyoka/detail/llvm_helpers.hpp>
 #include <heyoka/detail/logging_impl.hpp>
@@ -1598,12 +1599,7 @@ auto add_cfunc_impl(llvm_state &s, const std::string &name, const F &fn, std::ui
     // Append ".strided" to the function name.
     const auto sname = name + ".strided";
     // Now create the function.
-    auto *f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, sname, &md);
-    if (f == nullptr) {
-        // LCOV_EXCL_START
-        throw std::invalid_argument(fmt::format("Unable to create a compiled function with name '{}'", sname));
-        // LCOV_EXCL_STOP
-    }
+    auto *f = llvm_func_create(ft, llvm::Function::ExternalLinkage, sname, &md);
     // NOTE: a cfunc cannot call itself recursively.
     f->addFnAttr(llvm::Attribute::NoRecurse);
 
@@ -1661,12 +1657,7 @@ auto add_cfunc_impl(llvm_state &s, const std::string &name, const F &fn, std::ui
     fargs.pop_back();
     ft = llvm::FunctionType::get(builder.getVoidTy(), fargs, false);
     assert(ft != nullptr); // LCOV_EXCL_LINE
-    f = llvm::Function::Create(ft, llvm::Function::ExternalLinkage, name, &md);
-    if (f == nullptr) {
-        // LCOV_EXCL_START
-        throw std::invalid_argument(fmt::format("Unable to create a compiled function with name '{}'", name));
-        // LCOV_EXCL_STOP
-    }
+    f = llvm_func_create(ft, llvm::Function::ExternalLinkage, name, &md);
 
     // Set the names/attributes of the function arguments.
     out_ptr = f->args().begin();