Code refactoring wrt Metall macros

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.12)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 include(FetchContent)
 
@@ -82,33 +82,21 @@ option(JUST_INSTALL_METALL_HEADER "Just install Metall header files (do not buil
 option(BUILD_UTILITY "Build utility programs" OFF)
 option(BUILD_DOC "Build API documentation" OFF)
 option(BUILD_C "Build C examples and libraries" OFF)
-option(VERBOSE_SYSTEM_SUPPORT_WARNING "Show compile time warning regarding system support" OFF)
-option(DISABLE_FREE_FILE_SPACE "Disable freeing file space" OFF)
-option(DISABLE_SMALL_OBJECT_CACHE "Disable small object cache" OFF)
 option(BUILD_EXAMPLE "Build the examples" OFF)
-option(BUILD_BENCH "Build the benchmark" OFF)
 option(BUILD_TEST "Build the test" OFF)
 option(RUN_LARGE_SCALE_TEST "Run large scale tests" OFF)
 option(RUN_BUILD_AND_TEST_WITH_CI "Perform build and basic test with CI" OFF)
-option(BUILD_VERIFICATION "Build verification directory" OFF)
-option(USE_SORTED_BIN "Use VM space aware algorithm in the bin directory" OFF)
-
-set(DEFAULT_VM_RESERVE_SIZE "0" CACHE STRING
-        "Set the default VM reserve size (use the internally defined value if 0 is specified)")
-set(MAX_SEGMENT_SIZE "0" CACHE STRING
-        "Set the max segment size (use the internally defined value if 0 is specified)")
-set(INITIAL_SEGMENT_SIZE "0" CACHE STRING
-        "Set the initial segment size (use the internally defined value if 0 is specified)")
+option(BUILD_BENCH "Build the benchmark" OFF)
+option(BUILD_VERIFICATION "Build verification programs" OFF)
+set(COMPILER_DEFS "" CACHE STRING "A list of Metall compile definitions to be added to all targets")
 
 # ---------- Experimental options ---------- #
-option(USE_ANONYMOUS_NEW_MAP "Use the anonymous map when creating a new map region" OFF)
+#option(USE_ANONYMOUS_NEW_MAP "Use the anonymous map when creating a new map region" OFF)
 set(UMAP_ROOT "" CACHE PATH "UMap installed root directory")
 
 option(ONLY_DOWNLOAD_GTEST "Only downloading Google Test" OFF)
 option(SKIP_DOWNLOAD_GTEST "Skip downloading Google Test" OFF)
 option(BUILD_NUMA "Build programs that require the NUMA policy library (numa.h)" OFF)
-set(FREE_SMALL_OBJECT_SIZE_HINT "0" CACHE STRING
-        "Try to free the associated pages and file space when objects equal to or larger than that is deallocated")
 
 # -------------------------------------------------------------------------------- #
 
@@ -153,64 +141,23 @@ endif ()
 # -------------------------------------------------------------------------------- #
 # Executables
 # -------------------------------------------------------------------------------- #
+
+# ---------- Metall Macros ---------- #
+foreach(X ${COMPILER_DEFS})
+    message(STATUS "Metall compile definition: ${X}")
+endforeach()
+
+
+# ---------- CMAKE_BUILD_TYPE ---------- #
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
     message(STATUS "CMAKE_BUILD_TYPE is set as Release")
 endif ()
 
+
 # ---------- MPI ---------- #
 find_package(MPI)
 
-# ---------- Configure Metall ---------- #
-if (FREE_SMALL_OBJECT_SIZE_HINT GREATER 0)
-    list(APPEND METALL_DEFS "METALL_FREE_SMALL_OBJECT_SIZE_HINT=${FREE_SMALL_OBJECT_SIZE_HINT}")
-    message(STATUS "Try to free space for objects >= ${FREE_SMALL_OBJECT_SIZE_HINT} bytes")
-endif ()
-
-if (VERBOSE_SYSTEM_SUPPORT_WARNING)
-    list(APPEND METALL_DEFS "METALL_VERBOSE_SYSTEM_SUPPORT_WARNING")
-    message(STATUS "Show compile time warning regarding system support")
-endif ()
-
-if (DISABLE_FREE_FILE_SPACE)
-    list(APPEND METALL_DEFS "METALL_DISABLE_FREE_FILE_SPACE")
-    message(STATUS "Disable freeing file space in Metall")
-endif ()
-
-if (DISABLE_SMALL_OBJECT_CACHE)
-    list(APPEND METALL_DEFS "METALL_DISABLE_OBJECT_CACHE")
-    message(STATUS "Disable small object cache")
-endif ()
-
-if (DEFAULT_VM_RESERVE_SIZE GREATER 0)
-    list(APPEND METALL_DEFS "METALL_DEFAULT_VM_RESERVE_SIZE=${DEFAULT_VM_RESERVE_SIZE}")
-    message(STATUS "METALL_DEFAULT_VM_RESERVE_SIZE=${DEFAULT_VM_RESERVE_SIZE}")
-endif ()
-
-if (MAX_SEGMENT_SIZE GREATER 0)
-    list(APPEND METALL_DEFS "METALL_MAX_SEGMENT_SIZE=${MAX_SEGMENT_SIZE}")
-    message(STATUS "METALL_MAX_SEGMENT_SIZE=${MAX_SEGMENT_SIZE}")
-endif ()
-
-if (INITIAL_SEGMENT_SIZE GREATER 0)
-    list(APPEND METALL_DEFS "METALL_INITIAL_SEGMENT_SIZE=${INITIAL_SEGMENT_SIZE}")
-    message(STATUS "METALL_INITIAL_SEGMENT_SIZE=${INITIAL_SEGMENT_SIZE}")
-endif ()
-
-if (USE_SORTED_BIN)
-    list(APPEND METALL_DEFS "METALL_USE_SORTED_BIN")
-    message(STATUS "Use VM space aware algorithm in the bin directory")
-endif ()
-
-if (USE_ANONYMOUS_NEW_MAP)
-    if (USE_ANONYMOUS_NEW_MAP AND UMAP_ROOT)
-        message(FATAL_ERROR "USE_ANONYMOUS_NEW_MAP and UMAP_ROOT options cannot coexist")
-    endif ()
-
-    list(APPEND METALL_DEFS "METALL_USE_ANONYMOUS_NEW_MAP")
-    message(STATUS "Use the anonymous map for new map region")
-endif ()
-
 # Requirements for GCC
 if (NOT RUN_BUILD_AND_TEST_WITH_CI)
     if (("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU"))
@@ -304,8 +251,8 @@ function(common_setup_for_metall_executable name)
     # --------------------
 
     # ----- Compile Definitions ----- #
-    foreach(X IN LISTS METALL_DEFS)
-        target_compile_definitions(${name} PRIVATE ${X})
+    foreach(X ${COMPILER_DEFS})
+         target_compile_definitions(${name} PRIVATE ${X})
     endforeach()
     # --------------------
 
@@ -325,7 +272,7 @@ function(common_setup_for_metall_executable name)
         target_include_directories(${name} PRIVATE ${UMAP_ROOT}/include)
         if (LIBUMAP)
             target_link_libraries(${name} PRIVATE ${LIBUMAP})
-            target_compile_definitions(${name} PRIVATE METALL_USE_UMAP)
+            target_compile_definitions(${name} PRIVATE "METALL_USE_UMAP")
         endif ()
     endif ()
     # --------------------

docs/Doxyfile.in

@@ -835,8 +835,6 @@ INPUT                  = ../include/metall \
                          ../include/metall/json \
                          ../include/metall/container/experimental/jgraph \
                          ../include/metall/utility \
-                         ../include/metall/detail/base_stl_allocator.hpp \
-                         ../include/metall/detail/utility/named_proxy.hpp \
                          ../example/graph_data_structure \
                          ../example/json
 
@@ -2139,7 +2137,7 @@ ENABLE_PREPROCESSING   = YES
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
 # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
 # the macro expansion is limited to the macros specified with the PREDEFINED and

docs/readthedocs/advanced_build/cmake.md

@@ -17,7 +17,7 @@ cmake build_doc  # option; BUILD_DOC must be ON when running cmake
 
 ## Required
 
- - CMake 3.10 or more.
+ - CMake 3.12 or more.
  - GCC 8.1 or more.
  - Boost C++ Libraries 1.64 or more (build is not required; needs only their header files).
 

docs/readthedocs/basics/compile_time_options.md

@@ -1,26 +1,31 @@
-# Compile-time Options
+# Compile-Time Options
 
 There are some compile-time options (C/C++ macro) as follows to configure the behavior of Metall:
 
+- METALL_DEFAULT_CAPACITY=*bytes*
+  - The default capacity of a segment/datastore.
+  - This value is used when a user does not specify the capacity of a datastore when creating it.
+
+
+- METALL_SEGMENT_BLOCK_SIZE=*bytes*
+  - The segment block size used by the default segment storage manager.
+  - Metall allocates a backing file with this size.
+
 
 - METALL_DISABLE_FREE_FILE_SPACE
-    - If defined, Metall does not free file space
+  - If defined, Metall does not free file space.
 
 
-- METALL_DEFAULT_VM_RESERVE_SIZE=*bytes*
-    - The default virtual memory reserve size
-    - An internally defined value is used if 0 is specified
-    - Wll be rounded up to a multiple of the system page size (e.g., 4 KB) internally 
+- METALL_VERBOSE_SYSTEM_SUPPORT_WARNING
+  - If defined, Metall shows warning messages at compile time if the system does not support important features.
 
 
-- METALL_INITIAL_SEGMENT_SIZE=*bytes*
-    - The initial segment size
-    - Use the internally defined value if 0 is specified
-    - Wll be rounded up to a multiple of the system page size internally
+- METALL_DISABLE_CONCURRENCY
+  - Disable concurrency support in Metall. This option is useful when Metall is used in a single-threaded application.
+  - If this macro is defined, applications must not call Metall concurrently from multiple threads.
+  - Even if this option is enabled, Metall still uses multiple threads for background tasks, such as synchronizing segment files.
 
 
 - METALL_FREE_SMALL_OBJECT_SIZE_HINT=*bytes*
-    - Experimental option
-    - If defined, Metall tries to free space when an object equal or larger than specified bytes is deallocated
-    - Wll be rounded up to a multiple of the page size internally
-
+  - If defined, Metall tries to free space when an object equal to or larger than the specified bytes is deallocated.
+  - Wll be rounded up to a multiple of the page size internally.

include/metall/defs.hpp

@@ -0,0 +1,65 @@
+// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall
+// Project Developers. See the top-level COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+/// \file defs.hpp
+/// \brief Common definitions for Metall
+
+#ifndef METALL_DEFS_HPP
+#define METALL_DEFS_HPP
+
+/// \def METALL_MAX_CAPACITY
+/// The max capacity, i.e., the maximum total memory size a single Metall
+/// datastore can allocate. This value is a theoretical limit, and the actual
+/// limit is smaller than this value.
+#ifndef METALL_MAX_CAPACITY
+#define METALL_MAX_CAPACITY (1ULL << 48ULL)
+#endif
+
+#ifdef METALL_MAX_SEGMENT_SIZE
+#warning \
+    "METALL_MAX_SEGMENT_SIZE is deprecated. Use METALL_MAX_CAPACITY instead."
+#endif
+
+/// \def METALL_DEFAULT_CAPACITY
+/// The default Metall store capacity, i.e., the default maximum total memory
+/// size a single Metall datastore can allocate. This value is used when a new
+/// Metall datastore is created without the capacity parameter. This value is a
+/// theoretical limit, and the actual limit is smaller than this value. This
+/// value must be less than or equal to METALL_MAX_CAPACITY.
+#ifndef METALL_DEFAULT_CAPACITY
+#if defined(__linux__)
+#define METALL_DEFAULT_CAPACITY (1ULL << 43ULL)
+#else
+#define METALL_DEFAULT_CAPACITY (1ULL << 42ULL)
+#endif
+#endif
+
+#ifdef METALL_DEFAULT_VM_RESERVE_SIZE
+#warning \
+    "METALL_DEFAULT_VM_RESERVE_SIZE is deprecated. Use METALL_DEFAULT_CAPACITY instead."
+#endif
+
+/// \def METALL_SEGMENT_BLOCK_SIZE
+/// The segment block size the default segment storage use.
+#ifndef METALL_SEGMENT_BLOCK_SIZE
+#define METALL_SEGMENT_BLOCK_SIZE (1ULL << 28ULL)
+#endif
+
+#ifdef METALL_INITIAL_SEGMENT_SIZE
+#warning \
+    "METALL_INITIAL_SEGMENT_SIZE is deprecated. Use METALL_SEGMENT_BLOCK_SIZE instead."
+#endif
+
+/// \def METALL_DISABLE_CONCURRENCY
+/// Disable concurrency support in Metall. If this macro is defined,
+/// applications must not call Metall concurrently from multiple threads. This
+/// option is useful when Metall is used in a single-threaded application. Even
+/// if this option is enabled, Metall still uses multiple threads for background
+/// tasks, such as synchronizing segment files.
+#ifdef DOXYGEN_SKIP
+#define METALL_DISABLE_CONCURRENCY
+#endif
+
+#endif  // METALL_DEFS_HPP

include/metall/json/json_fwd.hpp

@@ -12,15 +12,15 @@
 #include <string>
 #include <variant>
 
-#if defined(DOXYGEN_SKIP)
+#ifdef DOXYGEN_SKIP
 /// \brief If defined, link with a buit Boost.JSON.
 #define METALL_LINK_WITH_BOOST_JSON
 
 /// \brief Include guard for boost/json/src.hpp
 #define METALL_BOOST_JSON_SRC_INCLUDED
 #endif
 
-#if METALL_LINK_WITH_BOOST_JSON
+#ifdef METALL_LINK_WITH_BOOST_JSON
 #include <boost/json.hpp>
 #else
 #ifndef METALL_BOOST_JSON_SRC_INCLUDED

include/metall/kernel/manager_kernel.hpp

@@ -23,7 +23,6 @@
 #include <metall/offset_ptr.hpp>
 #include <metall/version.hpp>
 #include <metall/kernel/manager_kernel_fwd.hpp>
-#include <metall/kernel/manager_kernel_defs.hpp>
 #include <metall/kernel/segment_header.hpp>
 #include <metall/kernel/segment_allocator.hpp>
 #include <metall/kernel/attributed_object_directory.hpp>
@@ -85,17 +84,26 @@ class manager_kernel {
   static constexpr const char *k_datastore_segment_dir_name = "segment";
 
   // For segment
+#ifndef METALL_DEFAULT_CAPACITY
+#error "METALL_DEFAULT_CAPACITY is not defined."
+#endif
   static constexpr size_type k_default_vm_reserve_size =
-      METALL_DEFAULT_VM_RESERVE_SIZE;
+      METALL_DEFAULT_CAPACITY;
   static_assert(k_chunk_size <= k_default_vm_reserve_size,
                 "Chunk size must be <= k_default_vm_reserve_size");
 
-  static constexpr size_type k_max_segment_size = METALL_MAX_SEGMENT_SIZE;
+#ifndef METALL_MAX_CAPACITY
+#error "METALL_MAX_CAPACITY is not defined."
+#endif
+  static constexpr size_type k_max_segment_size = METALL_MAX_CAPACITY;
   static_assert(k_default_vm_reserve_size <= k_max_segment_size,
                 "k_default_vm_reserve_size must be <= k_max_segment_size");
 
+#ifndef METALL_SEGMENT_BLOCK_SIZE
+#error "METALL_SEGMENT_BLOCK_SIZE is not defined."
+#endif
   static constexpr size_type k_initial_segment_size =
-      METALL_INITIAL_SEGMENT_SIZE;
+      METALL_SEGMENT_BLOCK_SIZE;
   static_assert(k_initial_segment_size <= k_default_vm_reserve_size,
                 "k_initial_segment_size must be <= k_default_vm_reserve_size");
   static_assert(k_chunk_size <= k_initial_segment_size,

include/metall/metall.hpp

@@ -6,6 +6,7 @@
 #ifndef METALL_METALL_HPP
 #define METALL_METALL_HPP
 
+#include <metall/defs.hpp>
 #include <metall/basic_manager.hpp>
 #include <metall/logger.hpp>
 #include <metall/version.hpp>