adding a doubling function to slab hash table for offline use; adding…

… metrics (#289)

* adding metrics to the hash module

* Squashed 'deps/ccommon/' changes from f5efe29..5e64de2

5e64de2 WIP: add fluxcapacitor support for linux build (#242)
55769cc restore cmake for rust libraries (#241)
6b1d8d5 Improve the Rust build story (#240)
5955d6b move nodelay setting to the server socket, and accepted connections will inherit that (#238)
aafd20f formatting (#237)
3b5c069 Update ccommon_rs to use bytes v5.0 (#235)
df74087 Address clippy lints in rust code (#234)
7c5bbd1 Make metrics and options Send + Sync (#233)
c2e617e Ensure that manually implemented C options use the correct name (#232)
6e76741 Add some docs
b5b4c5a Change check_pipe to use nanosleep instead of usleep (#231)
eb0a389 Use name of field instead of description for C metrics (#230)
16ddc76 Fix broken buf impls (#229)
eefcdcb Avoid redundant rebuilds of rust packages in CI (#228)
33f62a8 Update bindgen to also generate bindings recursively (#227)
27ffc7c Implement bytes::Buf and Bytes::ButMut on Buf and OwnedBuf (#226)
f873930 Various small bugfixes and usability improvements for rust code (#225)
37a1ecd Port option parsing module to Rust (#224)
38f7556 Fix failed test detection (#215)
0ab1604 Conditionally use std::any::type_name if it's supported (#223)
98176d3 Backport changes from #265 (#222)
ba54096 Remove test for removed rust logging functionality (#221)
519118d Rewrite cmake cargo build wrapper (#220)
5d23b3a Fix some small typos found in #263 (#219)
475dda7 Clean up logging shim (#218)
1d28dd2 Expand rust bindings to add options, metrics, and ccbox (#217)
88b5400 Upstream changes from #261 (#216)
4e99e63 fix a bug and change how check is found (#214)
54067ef slightly simplify accept error-handling logic (#210)
e9fe980 Fix synchronize ccommon with pelikan deps/ccommon (#212)
7eb6424 Cleanup libcheck related code (#211)
683bc1a cc_bstring simplify and fix (#207)
8737d99 continue on server socket on non-blocking errors (#209)
2a62281 add atoi64 to bstring (#206)
f71c657 cc_option simplify _allowed_in_name (#205)
24e3131 Add ITT instrumentation option (#204)
236c98d Fix docs (#200)
e58f6a8 cc_array and cc_ring_array NULL fixes (#201)
1c8df42 Add basic support of build type (#199)
7107988 Fix now_ns() (#198)
da240e5 cc: extend cc_util module (#196)
4846b15 Fix TAILQ_REINIT (#195)
4f5dbb0 Update Cmake version to 2.8 (#197)
2e6f78a cc_mm use OS_DARWIN macro to detect OS (#194)
57acaf6 cc: extend queue module (#193)
a64ada2 cc: extend duration module (#192)
b117632 reverting CMake file changes (#191)
dea5bee backport changes made to ccommon in pelikan (#190)
a4c0334 add linebreak to stats_log() (#188)
05eb03e fix inconsistent naming and bump version (#187)
4acc53a Stats to file (#186)
2168fec minimize osx build config (#185)
42b24de Simplify rust options, specify fewer output targets (#183)
c9fa905 update CMakeRust used to latest version, tweaks to make build work (#184)
2ef0163 Reorder dependency includes in cmake, don't parallel build (#182)
a6a54d9 remove endian-specific logic from str*cmp (#177)
4c0668b epoll_create* ignores size hint in newer kernels, switch to new API (#179)
c9c5ee5 improve cc_bstring string literal and cstring names (#176)
0184d73 Add unit tests for buffer, fix buf/dbuf bugs and refactor (#174)
d7dab43 create a .cargo/config so intellij uses the same target dir as cmake (#173)
e710712 use accept4 for tcp_accept when available (#171)
21ba10e Remove cargo lock for shared lib, closes #169 (#172)
24660f1 update style guide (#170)
17baf1e Per thread logging (#168)

git-subtree-dir: deps/ccommon
git-subtree-split: 5e64de297dbf1d49b59746431448f5f4dfb5d640

* clean up cmake file

* rename findcheck cmake module

* adding new metrics to rust wrapper

* add ci/cargo.sh to pelikan for consistent invocation

* Squashed 'deps/ccommon/' changes from 5e64de2..0db032a

0db032a tuning cmake / ci further (#243)
49dcaff improve cargo build script (#244)

git-subtree-dir: deps/ccommon
git-subtree-split: 0db032afcac0a75887281c3876ed9f7bb10391ce

* update ci based on changes first tested in ccommon

.travis.yml

@@ -1,6 +1,6 @@
 sudo: false
 language: c
-dist: xenial
+dist: bionic
 
 # using anchor to import sources into linux builds
 addons:
@@ -11,69 +11,47 @@ addons:
 # important for allowed-to-fail matching
 # see https://docs.travis-ci.com/user/customizing-the-build#Rows-that-are-Allowed-to-Fail
 env:
-  - ALLOWED_TO_FAIL=0
+  global:
+  - MAKEFLAGS="-j 2"
 
 # travis currently does not support directly setting gcc/clang with versions
 # (e.g. gcc-4.8) as value for the compiler key. So we will have to manually
 # request these packages and use environment varibles to create the matrix.
-matrix:
+jobs:
+  fast_finish: true
   include:
-    - name: "gcc-5 on Linux"
+    - name: "gcc-7 on Linux"
       compiler: gcc
-      addons:
-        apt:
-          <<: *apt
-          packages:
-            - libsubunit-dev
 
-    - name: "gcc-5 on Linux, Rust enabled"
+    - name: "gcc-7 on Linux, Rust enabled"
       compiler: gcc
       env:
         - RUST_ENABLED=1
-      addons:
-        apt:
-          <<: *apt
-          packages:
-            - libsubunit-dev
 
     - name: "cargo build"
       language: rust
-      addons:
-        apt:
-          <<: *apt
-          packages:
-            - libsubunit-dev
       script:
-        - cargo build
-        - cargo test
-        - cargo build --release
-        - cargo test --release
+        - ./ci/cargo.sh
 
     - os: osx
-      osx_image: xcode10.1
+      osx_image: xcode11.4
       compiler: clang
 
     - os: osx
-      osx_image: xcode10.1
+      osx_image: xcode11.4
       language: rust
       script:
-        - cargo build
-        - cargo test
-        - cargo build --release
-        - cargo test --release
+        - ./ci/cargo.sh
 
   allow_failures:
     - os: osx
-      osx_image: xcode10.1
+      osx_image: xcode11.4
       compiler: clang
     - os: osx
-      osx_image: xcode10.1
+      osx_image: xcode11.4
       language: rust
       script:
-        - cargo build
-        - cargo test
-        - cargo build --release
-        - cargo test --release
+        - ./ci/cargo.sh
 
 before_install:
   - ./ci/before-install.sh

CMakeLists.txt

@@ -1,7 +1,16 @@
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.0)
 project(pelikan C)
 
-enable_testing()
+# Uncomment the following to output dependency graph debugging messages
+# set_property(GLOBAL PROPERTY GLOBAL_DEPENDS_DEBUG_MODE 1)
+
+###################
+# detect platform #
+###################
+
+# TODO(yao):
+#   1. make this a .cmake macro and put it under cmake/
+#   2. avoid calling this twice when included by another project, e.g. Pelikan
 
 # detect platform
 macro(set_platform system_name)
@@ -21,20 +30,40 @@ if(OS_PLATFORM STREQUAL "OS_UNSUPPORTED")
     message(FATAL_ERROR "unsupported operating system")
 endif()
 
+####################
+# define variables #
+####################
+
 # the following sections work with config.h(.in): version, compile variables
 # config.h.in has to include entries set/tested here for them to have effect
 
 # version info
 set(${PROJECT_NAME}_VERSION_MAJOR 0)
 set(${PROJECT_NAME}_VERSION_MINOR 1)
 set(${PROJECT_NAME}_VERSION_PATCH 1)
-set(${PROJECT_NAME}_VERSION ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH})
+set(${PROJECT_NAME}_VERSION
+    ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH}
+    )
+set(${PROJECT_NAME}_RELEASE_VERSION
+   ${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}
+   )
 
 # flags => compile-time variables: use modules/macros
 option(HAVE_ASSERT_LOG "assert_log enabled by default" ON)
 option(HAVE_ASSERT_PANIC "assert_panic disabled by default" OFF)
 option(HAVE_LOGGING "logging enabled by default" ON)
 option(HAVE_STATS "stats enabled by default" ON)
+option(HAVE_TEST "test built by default" ON)
+option(HAVE_DEBUG_MM "debugging oriented memory management disabled by default" OFF)
+option(HAVE_COVERAGE "code coverage" OFF)
+option(HAVE_RUST "rust bindings not built by default" OFF)
+option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF)
+
+option(FORCE_CHECK_BUILD "Force building check with ci/install-check.sh" OFF)
+
+if(HAVE_RUST)
+    option(RUST_VERBOSE_BUILD "pass -vv to cargo compilation" OFF)
+endif()
 
 option(TARGET_PINGSERVER "build pingserver binary" ON)
 option(TARGET_RDS "build rich data server binary" ON)
@@ -45,25 +74,8 @@ option(TARGET_CDB "build cdb binary (implies HAVE_RUST)" OFF)
 option(TARGET_RESPCLI "build resp-cli binary" ON)
 option(TARGET_HTTP "build experimental twemcache-http server (implies HAVE_RUST)" OFF)
 
-option(HAVE_RUST "build features written in rust" OFF)
-option(RUST_USE_MUSL "build rust deps against musl" OFF)
-option(BUILD_AND_INSTALL_CHECK "build our own version of check and link against it" OFF)
 option(USE_PMEM "build persistent memory features" OFF)
-option(HAVE_ITT_INSTRUMENTATION "instrument code with ITT API" OFF)
-
-option(COVERAGE "code coverage" OFF)
 
-# Note: duplicate custom targets only works with Makefile generators, will break XCode & VS
-# reference: http://public.kitware.com/Bug/view.php?id=6348
-set_property(GLOBAL PROPERTY ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
-set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
-set(CMAKE_MACOSX_RPATH 1)
-
-# Required for properly linking with rust code
-set(CMAKE_POSITION_INDEPENDENT_CODE true)
-
-include(CheckSymbolExists)
-check_symbol_exists(sys_signame signal.h HAVE_SIGNAME)
 
 include(CheckFunctionExists)
 check_function_exists(backtrace HAVE_BACKTRACE)
@@ -74,6 +86,15 @@ configure_file(
   "${PROJECT_SOURCE_DIR}/cmake/config.h.in"
   "${PROJECT_BINARY_DIR}/config.h")
 
+# Note: duplicate custom targets only works with Makefile generators, will break XCode & VS
+# reference: http://public.kitware.com/Bug/view.php?id=6348
+set_property(GLOBAL PROPERTY ALLOW_DUPLICATE_CUSTOM_TARGETS 1)
+set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY true)
+set(CMAKE_MACOSX_RPATH 1)
+
+# Required for properly linking with rust code
+set(CMAKE_POSITION_INDEPENDENT_CODE true)
+
 # set compiler flags
 # string concat is easier in 3.0, but older versions don't have the concat subcommand
 # so we are using list as input until we move to new version
@@ -145,6 +166,48 @@ add_subdirectory(${CCOMMON_SOURCE_DIR} ${PROJECT_BINARY_DIR}/ccommon)
 include(FindPackageHandleStandardArgs)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake")
 
+# test dependencies
+if (HAVE_TEST)
+    enable_testing()
+    # first we try default ways of finding gmodules
+    find_package(CHECK)
+    if(CHECK_FOUND)
+        check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING)
+    endif(CHECK_FOUND)
+    # if we don't have a working version of check, build it
+    if(NOT CHECK_WORKING OR FORCE_CHECK_BUILD)
+        set(LIBCHECK_PREFIX "${CMAKE_BINARY_DIR}/check")
+        execute_process(
+            COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-check.sh" "${LIBCHECK_PREFIX}"
+            TIMEOUT 300  # if this doesn't build in 5 minutes something is hosed
+            RESULT_VARIABLE LIBCHECK_RETCODE
+        )
+        if(LIBCHECK_RETCODE) # non-zero means error
+            message(STATUS "build libcheck failed, return code: " ${LIBCHECK_RETCODE})
+        else(LIBCHECK_RETCODE)
+            # use locally built libcheck
+            set(CHECK_ROOT_DIR "${LIBCHECK_PREFIX}")
+            find_package(CHECK)
+        endif(LIBCHECK_RETCODE)
+    endif(NOT CHECK_WORKING OR FORCE_CHECK_BUILD)
+
+    # use fluxcapacitor to mock time
+    if(OS_PLATFORM STREQUAL "OS_LINUX")
+        set(FLUXCAP_PREFIX "${CMAKE_BINARY_DIR}/fluxcapacitor")
+        execute_process(
+            COMMAND "bash" "${PROJECT_SOURCE_DIR}/ci/install-fluxcapacitor.sh" "${FLUXCAP_PREFIX}"
+            TIMEOUT 60  # if this doesn't build in 60 seconds something is hosed
+            RESULT_VARIABLE FLUXCAP_RETCODE
+        )
+        if(FLUXCAP_RETCODE) # non-zero means error
+            message(STATUS "build fluxcapacitor failed, return code: " ${FLUXCAP_RETCODE})
+        else(FLUXCAP_RETCODE)
+            set(FLUXCAP_BINARY "${FLUXCAP_PREFIX}/fluxcapacitor")
+            message(STATUS "fluxcapacitor available at: " ${FLUXCAP_BINARY})
+        endif(FLUXCAP_RETCODE)
+    endif(OS_PLATFORM STREQUAL "OS_LINUX")
+endif(HAVE_TEST)
+
 find_package(PkgConfig QUIET)
 
 if (USE_PMEM)
@@ -167,17 +230,6 @@ if (HAVE_ITT_INSTRUMENTATION)
     link_libraries(${ITTNOTIFY_LIBRARIES})
 endif(HAVE_ITT_INSTRUMENTATION)
 
-find_package(Check)
-if(NOT CHECK_FOUND)
-    message(WARNING "Check is required to build and run tests")
-endif(NOT CHECK_FOUND)
-if(CHECK_FOUND)
-    check_symbol_exists(ck_assert_int_eq check.h CHECK_WORKING)
-    if(NOT CHECK_WORKING)
-        message(WARNING "Check version too old to build tests")
-    endif(NOT CHECK_WORKING)
-endif(CHECK_FOUND)
-
 find_package(Threads)
 
 if(TARGET_CDB)
@@ -202,30 +254,45 @@ include_directories(${include_directories}
 # server & (cli) client
 add_subdirectory(src)
 
-# tests: always build last
-if(CHECK_FOUND)
-    include_directories(${CHECK_INCLUDES})
-    link_directories(${CHECK_LIBRARY_DIRS})
+if(HAVE_TEST)
+    include_directories(${include_directories} ${CHECK_INCLUDES})
     add_subdirectory(test)
-endif(CHECK_FOUND)
+    if(${OS_PLATFORM} MATCHES "OS_LINUX")
+        add_subdirectory(benchmarks)
+    endif()
+endif(HAVE_TEST)
 
-if(${OS_PLATFORM} MATCHES "OS_LINUX")
-  add_subdirectory(benchmarks)
-endif()
 
-# print a summary
+###################
+# print a summary #
+###################
 
+message(STATUS "<<++=====------------------\\/------------------=====++>>")
+message(STATUS "<<++                 pelikan summary                 ++>>")
+message(STATUS "<<++=====------------------/\\------------------=====++>>")
+message(STATUS "=============CMake related=============")
 message(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE})
-
 message(STATUS "PLATFORM: " ${OS_PLATFORM})
-
 message(STATUS "CPPFLAGS: " ${CMAKE_CPP_FLAGS})
 message(STATUS "CFLAGS: " ${CMAKE_C_FLAGS})
+message(STATUS "=======================================")
 
-message(STATUS "HAVE_SIGNAME: " ${HAVE_SIGNAME})
-
+message(STATUS "=======Status of system features=======")
 message(STATUS "HAVE_BACKTRACE: " ${HAVE_BACKTRACE})
+message(STATUS "=======================================")
+
+message(STATUS "======Status of optional features======")
 message(STATUS "HAVE_RUST: " ${HAVE_RUST})
+message(STATUS "HAVE_ASSERT_LOG: " ${HAVE_ASSERT_LOG})
+message(STATUS "HAVE_ASSERT_PANIC: " ${HAVE_ASSERT_PANIC})
+message(STATUS "HAVE_LOGGING: " ${HAVE_LOGGING})
+message(STATUS "HAVE_STATS: " ${HAVE_STATS})
+message(STATUS "HAVE_ITT_INSTRUMENTATION: " ${HAVE_ITT_INSTRUMENTATION})
+message(STATUS "HAVE_DEBUG_MM: " ${HAVE_DEBUG_MM})
+message(STATUS "HAVE_TEST: " ${HAVE_TEST})
+message(STATUS "HAVE_COVERAGE: " ${HAVE_COVERAGE})
+message(STATUS "USE_PMEM: " ${USE_PMEM})
+message(STATUS "=======================================")
 
 if(DUMP_ALL)
     message(STATUS "<<++=====------------------\\/------------------=====++>>")

ci/cargo.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cargo build
+cargo test
+cargo build --release
+cargo test --release

ci/install-check.sh

@@ -17,20 +17,42 @@ TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir"
 cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; }
 trap cleanup EXIT
 
-realpath() { python -c "from __future__ import print_function; import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; }
+realpath() { python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1"; }
 
 TOPLEVEL="$(cd "$(dirname "$(realpath "$0" >/dev/null || exit 1)")" && git rev-parse --show-toplevel)" || die 'failed to find TOPLEVEL'
 
-CHECK_VERSION=0.12.0
+
+CHECK_VERSION="0.14.0"
 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz"
 CHECK_DIR="check-${CHECK_VERSION}"
+CHECK_LOG="build-check.log"
+
+echo "building and installing check" >&2
+
+echo "building and installing check" >&2
 
 (
   cd "$TEMP" &&
     wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" &&
-    tar xvfz "${CHECK_TARBALL}" &&
+    tar xfz "${CHECK_TARBALL}" &&
     cd "${CHECK_DIR}" &&
-    ./configure --prefix="$CHECK_PREFIX" &&
-    make &&
+    mkdir build &&
+    cd build &&
+    cmake -DCMAKE_INSTALL_PREFIX="${CHECK_PREFIX}" .. &&
+    make -j &&
     make install
-) || die "check build failed"
+) >$TEMP/${CHECK_LOG} 2>&1
+
+RESULT=$?
+if [[ $RESULT -ne 0 ]]; then
+  cat >&2 <<EOS
+check build failed! log below:
+
+EOS
+
+  cat $TEMP/${CHECK_LOG}
+else
+  echo "Success!" >&2
+fi
+
+exit $RESULT