ci: run fuzz tests in parallel and generate coverage report (#4960)

CMakeLists.txt

@@ -246,7 +246,7 @@ if (COVERAGE)
     # on LLVM compilers. GCC would fail with "unrecognized compile options"
     # on -fprofile-instr-generate -fcoverage-mapping flags.
     if (NOT ${CMAKE_C_COMPILER_ID} MATCHES Clang)
-        message(FATAL_ERROR "This project requires clang for coverage support")
+        message(FATAL_ERROR "This project requires clang for coverage support. You are currently using " ${CMAKE_C_COMPILER_ID})
     endif()
     target_compile_options(${PROJECT_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping)
     target_link_options(${PROJECT_NAME} PUBLIC -fprofile-instr-generate -fcoverage-mapping)
@@ -667,7 +667,6 @@ if (BUILD_TESTING)
     if(S2N_FUZZ_TEST)
         message(STATUS "Fuzz build enabled")
         set(SCRIPT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/runFuzzTest.sh")
-        set(BUILD_DIR_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build")
         file(GLOB FUZZ_TEST_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/*.c")
 
         file(GLOB TESTLIB_SRC "tests/testlib/*.c")
@@ -684,18 +683,6 @@ if (BUILD_TESTING)
             set(FUZZ_TIMEOUT_SEC 60)
         endif()
 
-        if(DEFINED ENV{CORPUS_UPLOAD_LOC})
-            set(CORPUS_UPLOAD_LOC $ENV{CORPUS_UPLOAD_LOC})
-        else()
-            set(CORPUS_UPLOAD_LOC "none")
-        endif()
-
-        if(DEFINED ENV{ARTIFACT_UPLOAD_LOC})
-            set(ARTIFACT_UPLOAD_LOC $ENV{ARTIFACT_UPLOAD_LOC})
-        else()
-            set(ARTIFACT_UPLOAD_LOC "none")
-        endif()
-
         # Build LD_PRELOAD shared libraries
         file(GLOB LIBRARY_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz/LD_PRELOAD/*.c")
         foreach(SRC ${LIBRARY_SRCS})
@@ -729,9 +716,7 @@ if (BUILD_TESTING)
                     bash ${SCRIPT_PATH} 
                     ${TEST_NAME} 
                     ${FUZZ_TIMEOUT_SEC} 
-                    ${BUILD_DIR_PATH} 
-                    ${CORPUS_UPLOAD_LOC} 
-                    ${ARTIFACT_UPLOAD_LOC}
+                    ${CMAKE_CURRENT_SOURCE_DIR}
                 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests/fuzz
             )
             set_property(TEST ${TEST_NAME} PROPERTY LABELS "fuzz")

codebuild/bin/fuzz_corpus_download.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+for FUZZ_TEST in tests/fuzz/*.c; do
+    # extract file name without extension
+    TEST_NAME=$(basename "$FUZZ_TEST")
+    TEST_NAME="${TEST_NAME%.*}"
+
+    # temp corpus folder to store downloaded corpus files
+    TEMP_CORPUS_DIR="./tests/fuzz/temp_corpus_${TEST_NAME}"
+
+    # Check if corpus.zip exists in the specified S3 location.
+    # `> /dev/null 2>&1` redirects output to /dev/null.
+    # If the file is not found, `aws s3 ls` returns a non-zero exit code.
+    if aws s3 ls "s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip" > /dev/null 2>&1; then
+        aws s3 cp "s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip" "${TEMP_CORPUS_DIR}/corpus.zip"
+        unzip -o "${TEMP_CORPUS_DIR}/corpus.zip" -d "${TEMP_CORPUS_DIR}" > /dev/null 2>&1
+    else
+        printf "corpus.zip not found for ${TEST_NAME}"
+    fi
+done

codebuild/bin/fuzz_corpus_upload.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+for FUZZ_TEST in tests/fuzz/*.c; do
+    # extract file name without extension
+    TEST_NAME=$(basename "$FUZZ_TEST")
+    TEST_NAME="${TEST_NAME%.*}"
+
+    # Upload generated corpus files to the S3 bucket.
+    zip -r ./tests/fuzz/corpus/${TEST_NAME}.zip ./tests/fuzz/corpus/${TEST_NAME}/ > /dev/null 2>&1
+    aws s3 cp ./tests/fuzz/corpus/${TEST_NAME}.zip s3://s2n-tls-fuzz-corpus/${TEST_NAME}/corpus.zip
+done
+

codebuild/bin/fuzz_coverage_report.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+
+set -e
+
+usage() {
+    echo "Usage: fuzz_coverage_report.sh"
+    exit 1
+}
+
+if [ "$#" -ne "0" ]; then
+    usage
+fi
+
+FUZZ_TEST_DIR="tests/fuzz"
+FUZZCOV_SOURCES="api bin crypto error stuffer tls utils"
+
+# generate coverage report for each fuzz test
+printf "Generating coverage reports... \n"
+
+mkdir -p coverage/fuzz
+for FUZZ_TEST in "$FUZZ_TEST_DIR"/*.c; do
+    # extract file name without extension
+    TEST_NAME=$(basename "$FUZZ_TEST")
+    TEST_NAME="${TEST_NAME%.*}"
+
+    # merge multiple .profraw files into a single .profdata file
+    llvm-profdata merge \
+        -sparse tests/fuzz/profiles/${TEST_NAME}/*.profraw \
+        -o tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata
+
+    # generate a coverage report in text format
+    llvm-cov report \
+        -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+        -show-functions \
+        > coverage/fuzz/${TEST_NAME}_cov.txt
+
+    # exports coverage data in LCOV format
+    llvm-cov export \
+        -instr-profile=tests/fuzz/profiles/${TEST_NAME}/${TEST_NAME}.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+        -format=lcov \
+        > coverage/fuzz/${TEST_NAME}_cov.info
+
+    # convert to HTML format
+    genhtml -q -o coverage/html/${TEST_NAME} coverage/fuzz/${TEST_NAME}_cov.info > /dev/null 2>&1
+done
+
+# merge all coverage reports into a single report that shows total s2n coverage
+printf "Calculating total s2n coverage... \n"
+llvm-profdata merge \
+    -sparse tests/fuzz/profiles/*/*.profdata \
+    -o tests/fuzz/profiles/merged_fuzz.profdata
+
+llvm-cov report \
+    -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+    > s2n_fuzz_coverage.txt
+
+llvm-cov export \
+    -instr-profile=tests/fuzz/profiles/merged_fuzz.profdata build/lib/libs2n.so ${FUZZCOV_SOURCES} \
+    -format=lcov \
+    > s2n_fuzz_cov.info
+
+genhtml s2n_fuzz_cov.info --branch-coverage -q -o coverage/fuzz/total_fuzz_coverage

codebuild/spec/buildspec_fuzz.yml

@@ -33,4 +33,4 @@ phases:
     on-failure: ABORT
     commands:
       # -L: Restrict tests to names matching the pattern 'fuzz'
-      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure"
+      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc)"

codebuild/spec/buildspec_fuzz_scheduled.yml

@@ -13,55 +13,10 @@
 # limitations under the License.
 version: 0.2
 
-batch:
-  build-matrix:
-    static:
-      env:
-        privileged-mode: true
-    dynamic:
-      env:
-        compute-type:
-          - BUILD_GENERAL1_LARGE
-        image:
-          - 024603541914.dkr.ecr.us-west-2.amazonaws.com/docker:ubuntu22codebuild
-        privileged-mode: true
-        variables:
-          S2N_LIBCRYPTO: 
-            - awslc
-          FUZZ_TESTS:
-            - "s2n_cert_req_recv_test"
-            - "s2n_certificate_extensions_parse_test"
-            - "s2n_client_ccs_recv_test"
-            - "s2n_client_cert_recv_test"
-            - "s2n_client_cert_verify_recv_test"
-            - "s2n_client_finished_recv_test"
-            - "s2n_client_fuzz_test"
-            - "s2n_client_hello_recv_fuzz_test"
-            - "s2n_client_key_recv_fuzz_test"
-            - "s2n_deserialize_resumption_state_test"
-            - "s2n_encrypted_extensions_recv_test"
-            - "s2n_extensions_client_key_share_recv_test"
-            - "s2n_extensions_client_supported_versions_recv_test"
-            - "s2n_extensions_server_key_share_recv_test"
-            - "s2n_extensions_server_supported_versions_recv_test"
-            - "s2n_hybrid_ecdhe_kyber_r3_fuzz_test"
-            - "s2n_kyber_r3_recv_ciphertext_fuzz_test"
-            - "s2n_kyber_r3_recv_public_key_fuzz_test"
-            - "s2n_memory_leak_negative_test"
-            - "s2n_openssl_diff_pem_parsing_test"
-            - "s2n_recv_client_supported_groups_test"
-            - "s2n_select_server_cert_test"
-            - "s2n_server_ccs_recv_test"
-            - "s2n_server_cert_recv_test"
-            - "s2n_server_extensions_recv_test"
-            - "s2n_server_finished_recv_test"
-            - "s2n_server_fuzz_test"
-            - "s2n_server_hello_recv_test"
-            - "s2n_stuffer_pem_fuzz_test"
-            - "s2n_tls13_cert_req_recv_test"
-            - "s2n_tls13_cert_verify_recv_test"
-            - "s2n_tls13_client_finished_recv_test"
-            - "s2n_tls13_server_finished_recv_test"
+env:
+  variables:
+    S2N_LIBCRYPTO: "awslc"
+    COMPILER: clang
 
 phases:
   pre_build:
@@ -76,13 +31,23 @@ phases:
       - |
         cmake . -Bbuild \
         -DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
+        -DCMAKE_C_COMPILER=/usr/bin/$COMPILER \
         -DS2N_FUZZ_TEST=on \
-        -DFUZZ_TIMEOUT_SEC=27000
+        -DCOVERAGE=on \
+        -DBUILD_SHARED_LIBS=on
       - cmake --build ./build -- -j $(nproc)
   post_build:
     on-failure: ABORT
     commands:
+      - ./codebuild/bin/fuzz_corpus_download.sh
       # -L: Restrict tests to labels matching the pattern 'fuzz'
-      # -R: Run the single fuzz test defined in ${FUZZ_TESTS}
       # --timeout: override ctest's default timeout of 1500
-      - cmake --build build/ --target test -- ARGS="-L fuzz -R ${FUZZ_TESTS} --output-on-failure --timeout 28800"
+      - cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure -j $(nproc) --timeout 28800"
+      - ./codebuild/bin/fuzz_corpus_upload.sh
+      - ./codebuild/bin/fuzz_coverage_report.sh
+
+artifacts:
+  # upload all files in the fuzz_coverage_report directory
+  files:
+    - '**/*'
+  base-directory: coverage/fuzz/total_fuzz_coverage

tests/fuzz/Readme.md

@@ -29,23 +29,39 @@ cmake --build build/ --target test -- ARGS="-L fuzz -R s2n_client_fuzz_test --ou
     2. If the test ends with `*_negative_test.c` the test is expected to fail in some way or return a non-zero integer (hereafter referred to as a "Negative test").
 2. Strive to be deterministic (Eg. shouldn't depend on the time or on the output of a RNG). Each test should either always pass if a Positive Test, or always fail if a Negative Test.
 3. If a Positive Fuzz test, it should have a non-empty corpus directory with inputs that have a relatively high branch coverage.
-4. Have a function `int s2n_fuzz_init(int *argc, char **argv[])` that will perform any initialization that will be run only once at startup.
-5. Have a function `int s2n_fuzz_test(const uint8_t *buf, size_t len)` that will pass `buf` to one of s2n's API's
-5. Optionally add a function `void s2n_fuzz_cleanup()` which cleans up any global state.
-6. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target
+4. If a Positive Fuzz test, define target functions for the test by adding following lines to your test below the copyright notice:
+> /* Target Functions: function1 function2 function3 */
+5. Have a function `int s2n_fuzz_init(int *argc, char **argv[])` that will perform any initialization that will be run only once at startup.
+6. Have a function `int s2n_fuzz_test(const uint8_t *buf, size_t len)` that will pass `buf` to one of s2n's API's
+7. Optionally add a function `void s2n_fuzz_cleanup()` which cleans up any global state.
+8. Call `S2N_FUZZ_TARGET(s2n_fuzz_init, s2n_fuzz_test, s2n_fuzz_cleanup)` at the bottom of the test to initialize the fuzz target
 
 ## Fuzz Test Coverage
-To generate coverage reports for fuzz tests, simply set the FUZZ_COVERAGE environment variable to any non-null value and run `make fuzz`. This will report the target function coverage and overall S2N coverage when running the tests. In order to define target functions for a fuzz test, simply add the following line to your fuzz test below the copyright notice:
+We run fuzz tests daily, with corpus files that are continuously being improved. Current coverage information can be view [here](https://dx1inn44oyl7n.cloudfront.net/fuzz-coverage-report/index.html).
 
-> /* Target Functions: function1 function2 function3 */
+To generate coverage reports for fuzz tests, s2n-tls needs to be compiled with the following options:
+```
+cmake . -Bbuild \
+-DCMAKE_PREFIX_PATH=/usr/local/$S2N_LIBCRYPTO \
+-DS2N_FUZZ_TEST=on \
+-DCOVERAGE=on \
+-DBUILD_SHARED_LIBS=on
 
-As the tests run, more detailed coverage reports are placed in the following directory:
+cmake --build ./build -- -j $(nproc)
+```
 
-> s2n/coverage/fuzz
+Next, run fuzz tests. This generates `.info` files for each fuzz test containing coverage information. 
+```
+cmake --build build/ --target test -- ARGS="-L fuzz --output-on-failure"
+```
 
-Each test outputs an HTML file which displays line by line coverage statistics and a .txt report which gives per-function coverage statistics in human-readable ASCII. After all fuzz tests have ran, a matching pair of coverage reports is generated for the total coverage of S2N by the entire set of tests performed.
+The `.info` files contain raw coverage data.  To convert them into HTML format, run the following script from the root of s2n-tls. This generates HTML files showing line-by-line coverage statistics for each fuzz test, as well as total coverage report for s2n-tls across all tests.
+```
+./codebuild/bin/fuzz_coverage_report.sh
+```
 
-Currently, this option isn't enabled for cmake build. See [#4748](https://github.com/aws/s2n-tls/issues/4748).
+You will see coverage reports placed in the following directory:
+> s2n-tls/tests/fuzz/coverage
 
 ## Fuzz Test Directory Structure
 For a test with name `$TEST_NAME`, its files should be laid out with the following structure:
@@ -62,9 +78,7 @@ For a test with name `$TEST_NAME`, its files should be laid out with the followi
 # Corpus
 A Corpus is a directory of "interesting" inputs that result in a good branch/code coverage. These inputs will be permuted in random ways and checked to see if this permutation results in greater branch coverage or in a failure (Segfault, Memory Leak, Buffer Overflow, Non-zero return code, etc). If the permutation results in greater branch coverage, then it will be added to the Corpus directory. If a Memory leak or a Crash is detected, that file will **not** be added to the corpus for that test, and will instead be written to the current directory (`s2n/tests/fuzz/crash-*` or `s2n/tests/fuzz/leak-*`). These files will be automatically deleted for any Negative Fuzz tests that are expected to crash or leak memory so as to not clutter the directory.
 
-To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing branch coverage over time. This process allows for gradual and automated enhancement of the corpus.
-
-To enable this, two environment variables must be defined: `CORPUS_UPLOAD_LOC` and `ARTIFACT_UPLOAD_LOC`. `CORPUS_UPLOAD_LOC` specifies where corpus files are stored, while `ARTIFACT_UPLOAD_LOC`defines where output logs from fuzzing are saved, which can be used for debugging if a new bug is detected during fuzzing.
+To continuously improve corpus inputs, we have a scheduled job that runs every day for approximately 8 hours. These tests begin with corpus files stored in an S3 bucket. At the end of each run, the existing corpus files are replaced with updated ones, potentially increasing coverage over time. This process allows for gradual and automated enhancement of the corpus.
 
 # LD_PRELOAD
 The `LD_PRELOAD` directory contains function overrides for each Fuzz test that will be used **instead** of the original functions defined elsewhere. These function overrides will only be used during fuzz tests, and will not effect the rest of the s2n codebase when not fuzzing. Using `LD_PRELOAD` instead of C Preprocessor `#ifdef`'s is preferable in the following ways: