From 914b07762ddf773e120d655f601484f220e8ea38 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 9 Sep 2019 14:44:30 -0400 Subject: [PATCH 01/73] Create "release-build-test" --- tests/release-build.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 tests/release-build.sh diff --git a/tests/release-build.sh b/tests/release-build.sh new file mode 100755 index 00000000000..deec2d3bb04 --- /dev/null +++ b/tests/release-build.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e +# test name +echo '' +echo ' ##### Release Build Test #####' +echo '' +# environment +[[ -z "$EOSIO_ROOT" && $(git --version) ]] && export EOSIO_ROOT="$(git rev-parse --show-toplevel)" +[[ -z "$EOSIO_ROOT" ]] && export EOSIO_ROOT="$(echo $(pwd)/ | grep -ioe '.*/eos/' -e '.*/eosio/' -e '.*/build/' | sed 's,/build/,/,')" +[[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" +# test +./nodeos --config-dir $(pwd)/config --data-dir $(pwd)/data & # run nodeos in background +sleep 10 +kill $! # kill nodeos gracefully, by PID +if [[ "$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" == '00' ]]; then + echo 'pass' + exit 0 +fi +echo 'FAIL' +exit 1 \ No newline at end of file From 395652d9264d96414aa0c890cfc3df6d89173b31 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 9 Sep 2019 17:02:32 -0400 Subject: [PATCH 02/73] Add test name, purpose, and description --- tests/release-build.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index deec2d3bb04..4e8a702d3eb 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -1,8 +1,15 @@ #!/bin/bash set -e -# test name +# test name and purpose echo '' -echo ' ##### Release Build Test #####' +echo ' ##### Release Build Test #####' +echo '' +echo 'The purpose of this test is to ensure that nodeos was built without debugging' +echo 'symbols. Debugging symbols enable software engineers to inspect and control a' +echo 'running program with a debugging tool, but they significantly slow down' +echo 'performance-critical applications like nodeos. Anyone intending to build and' +echo 'install nodeos from source should perform a "release build," which excludes' +echo 'debugging symbols to generate faster and lighter binaries.' echo '' # environment [[ -z "$EOSIO_ROOT" && $(git --version) ]] && export EOSIO_ROOT="$(git rev-parse --show-toplevel)" From 639c0fc500f707eccddae13307709ae43a2df415 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 9 Sep 2019 17:06:08 -0400 Subject: [PATCH 03/73] Add descriptive pass/fail statements --- tests/release-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 4e8a702d3eb..79c24e11b47 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -20,8 +20,8 @@ echo '' sleep 10 kill $! # kill nodeos gracefully, by PID if [[ "$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" == '00' ]]; then - echo 'pass' + echo 'PASS: Debug byte not set.' exit 0 fi -echo 'FAIL' +echo 'FAIL: Debug byte is set!' exit 1 \ No newline at end of file From 1b6a904085e70c594e78efdf4b427051ea2cc3bb Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 9 Sep 2019 17:12:31 -0400 Subject: [PATCH 04/73] Print debug byte on failure --- tests/release-build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 79c24e11b47..ef0135ebc6b 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -19,9 +19,11 @@ echo '' ./nodeos --config-dir $(pwd)/config --data-dir $(pwd)/data & # run nodeos in background sleep 10 kill $! # kill nodeos gracefully, by PID -if [[ "$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" == '00' ]]; then +export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" +if [[ "$DEBUG_BYTE" == '00' ]]; then echo 'PASS: Debug byte not set.' exit 0 fi echo 'FAIL: Debug byte is set!' +echo "Debug Byte = 0x$DEBUG_BYTE" exit 1 \ No newline at end of file From c994ad2855a63e9467dfdadbf31fdff481b9069b Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 9 Sep 2019 17:32:12 -0400 Subject: [PATCH 05/73] Print first kB of shared_memory.bin on failure --- tests/release-build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/release-build.sh b/tests/release-build.sh index ef0135ebc6b..836022bb5d8 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -26,4 +26,7 @@ if [[ "$DEBUG_BYTE" == '00' ]]; then fi echo 'FAIL: Debug byte is set!' echo "Debug Byte = 0x$DEBUG_BYTE" +echo 'First kilobyte of shared_memory.bin:' +echo '$ xxd -l 1024 shared_memory.bin' +xxd -l 1024 data/state/shared_memory.bin exit 1 \ No newline at end of file From 22d7340c043b3c2282d024771daffea4a218f24d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 9 Sep 2019 17:33:13 -0400 Subject: [PATCH 06/73] Print useful error message if nodeos binary is not found --- tests/release-build.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 836022bb5d8..7cbaae31d91 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -14,6 +14,16 @@ echo '' # environment [[ -z "$EOSIO_ROOT" && $(git --version) ]] && export EOSIO_ROOT="$(git rev-parse --show-toplevel)" [[ -z "$EOSIO_ROOT" ]] && export EOSIO_ROOT="$(echo $(pwd)/ | grep -ioe '.*/eos/' -e '.*/eosio/' -e '.*/build/' | sed 's,/build/,/,')" +if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/nodeos/nodeos" ]]; then + echo 'ERROR: nodeos binary not found!' + echo 'Looked in the following places:' + echo "$ ls -la $EOSIO_ROOT/build/bin" + ls -la $EOSIO_ROOT/build/bin + echo "$ ls -la $EOSIO_ROOT/build/programs/nodeos" + ls -la $EOSIO_ROOT/build/programs/nodeos + echo "Release Build Test not run because test conditions were not met." + exit 1 +fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" # test ./nodeos --config-dir $(pwd)/config --data-dir $(pwd)/data & # run nodeos in background @@ -29,4 +39,4 @@ echo "Debug Byte = 0x$DEBUG_BYTE" echo 'First kilobyte of shared_memory.bin:' echo '$ xxd -l 1024 shared_memory.bin' xxd -l 1024 data/state/shared_memory.bin -exit 1 \ No newline at end of file +exit 2 \ No newline at end of file From 14ab92f8f85ca5956432f39429ddef0b7ced4158 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:00:04 -0400 Subject: [PATCH 07/73] Clearer comments --- tests/release-build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 7cbaae31d91..ddebdace42a 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -25,10 +25,11 @@ if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/no exit 1 fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" -# test +# setup ./nodeos --config-dir $(pwd)/config --data-dir $(pwd)/data & # run nodeos in background sleep 10 kill $! # kill nodeos gracefully, by PID +# test export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then echo 'PASS: Debug byte not set.' From eb74195f31f589247185c1c961b0f263a4b3ee92 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:02:56 -0400 Subject: [PATCH 08/73] Print useful error if shared_memory.bin is not found --- tests/release-build.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index ddebdace42a..f02d2106529 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -29,6 +29,14 @@ fi ./nodeos --config-dir $(pwd)/config --data-dir $(pwd)/data & # run nodeos in background sleep 10 kill $! # kill nodeos gracefully, by PID +if [[ ! -f data/state/shared_memory.bin ]]; then + echo 'ERROR: nodeos state not found!' + echo 'Looked for shared_memory.bin in the following places:' + echo "$ ls -la $(pwd)/data/state" + ls -la $(pwd)/data/state + echo "Release Build Test not run because test setup failed." + exit 2 +fi # test export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then @@ -40,4 +48,4 @@ echo "Debug Byte = 0x$DEBUG_BYTE" echo 'First kilobyte of shared_memory.bin:' echo '$ xxd -l 1024 shared_memory.bin' xxd -l 1024 data/state/shared_memory.bin -exit 2 \ No newline at end of file +exit 3 \ No newline at end of file From 10b3ca908c30a2877e61defa638c21febe6f7eb0 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:06:13 -0400 Subject: [PATCH 09/73] Use quotes in case paths have spaces --- tests/release-build.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index f02d2106529..9f3025c2424 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -17,23 +17,23 @@ echo '' if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/nodeos/nodeos" ]]; then echo 'ERROR: nodeos binary not found!' echo 'Looked in the following places:' - echo "$ ls -la $EOSIO_ROOT/build/bin" - ls -la $EOSIO_ROOT/build/bin - echo "$ ls -la $EOSIO_ROOT/build/programs/nodeos" - ls -la $EOSIO_ROOT/build/programs/nodeos + echo "$ ls -la \"$EOSIO_ROOT/build/bin\"" + ls -la "$EOSIO_ROOT/build/bin" + echo "$ ls -la \"$EOSIO_ROOT/build/programs/nodeos\"" + ls -la "$EOSIO_ROOT/build/programs/nodeos" echo "Release Build Test not run because test conditions were not met." exit 1 fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" # setup -./nodeos --config-dir $(pwd)/config --data-dir $(pwd)/data & # run nodeos in background +./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" & # run nodeos in background sleep 10 kill $! # kill nodeos gracefully, by PID if [[ ! -f data/state/shared_memory.bin ]]; then echo 'ERROR: nodeos state not found!' echo 'Looked for shared_memory.bin in the following places:' - echo "$ ls -la $(pwd)/data/state" - ls -la $(pwd)/data/state + echo "$ ls -la \"$(pwd)/data/state\"" + ls -la "$(pwd)/data/state" echo "Release Build Test not run because test setup failed." exit 2 fi From 2f484106d7cf8f77f4f706b8661b861ad0c8b2c5 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:07:36 -0400 Subject: [PATCH 10/73] Better strings --- tests/release-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 9f3025c2424..29c09abed7a 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -21,7 +21,7 @@ if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/no ls -la "$EOSIO_ROOT/build/bin" echo "$ ls -la \"$EOSIO_ROOT/build/programs/nodeos\"" ls -la "$EOSIO_ROOT/build/programs/nodeos" - echo "Release Build Test not run because test conditions were not met." + echo 'Release build test not run.' exit 1 fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" @@ -34,7 +34,7 @@ if [[ ! -f data/state/shared_memory.bin ]]; then echo 'Looked for shared_memory.bin in the following places:' echo "$ ls -la \"$(pwd)/data/state\"" ls -la "$(pwd)/data/state" - echo "Release Build Test not run because test setup failed." + echo 'Release build test not run.' exit 2 fi # test From 6b4323b7d9278b2f9befe3c21df3a46ae1845a3d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:11:16 -0400 Subject: [PATCH 11/73] Remove "set -e" --- tests/release-build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 29c09abed7a..39daa610af5 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -e # test name and purpose echo '' echo ' ##### Release Build Test #####' From 61b1a527a32fc6c640482c10362c995b89d625bc Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:14:05 -0400 Subject: [PATCH 12/73] Even clearer comments --- tests/release-build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 39daa610af5..919982481fc 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -10,7 +10,7 @@ echo 'performance-critical applications like nodeos. Anyone intending to build a echo 'install nodeos from source should perform a "release build," which excludes' echo 'debugging symbols to generate faster and lighter binaries.' echo '' -# environment +# find nodeos [[ -z "$EOSIO_ROOT" && $(git --version) ]] && export EOSIO_ROOT="$(git rev-parse --show-toplevel)" [[ -z "$EOSIO_ROOT" ]] && export EOSIO_ROOT="$(echo $(pwd)/ | grep -ioe '.*/eos/' -e '.*/eosio/' -e '.*/build/' | sed 's,/build/,/,')" if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/nodeos/nodeos" ]]; then @@ -24,8 +24,8 @@ if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/no exit 1 fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" -# setup -./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" & # run nodeos in background +# run nodeos to generate state files +./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" & sleep 10 kill $! # kill nodeos gracefully, by PID if [[ ! -f data/state/shared_memory.bin ]]; then @@ -36,7 +36,7 @@ if [[ ! -f data/state/shared_memory.bin ]]; then echo 'Release build test not run.' exit 2 fi -# test +# test state files for debug flag export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then echo 'PASS: Debug byte not set.' From 43d6a34d2553d1581ed03526ab5aaa3b330eced5 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:21:08 -0400 Subject: [PATCH 13/73] Clean up after nodeos on test pass --- tests/release-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/release-build.sh b/tests/release-build.sh index 919982481fc..1b5ba514b54 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -40,6 +40,7 @@ fi export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then echo 'PASS: Debug byte not set.' + rm -rf config data exit 0 fi echo 'FAIL: Debug byte is set!' From 8db235104dd46f9ba4d5c4dd4ee88d147f9f5f2a Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:22:07 -0400 Subject: [PATCH 14/73] Clean up after nodeos on test fail --- tests/release-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/release-build.sh b/tests/release-build.sh index 1b5ba514b54..0c00522e31b 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -34,6 +34,7 @@ if [[ ! -f data/state/shared_memory.bin ]]; then echo "$ ls -la \"$(pwd)/data/state\"" ls -la "$(pwd)/data/state" echo 'Release build test not run.' + rm -rf config data exit 2 fi # test state files for debug flag @@ -48,4 +49,5 @@ echo "Debug Byte = 0x$DEBUG_BYTE" echo 'First kilobyte of shared_memory.bin:' echo '$ xxd -l 1024 shared_memory.bin' xxd -l 1024 data/state/shared_memory.bin +rm -rf config data exit 3 \ No newline at end of file From a827ad6840779be764b4f0528752bc960c0e3e9a Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:27:11 -0400 Subject: [PATCH 15/73] Silence nodeos --- tests/release-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 0c00522e31b..06dc93d4189 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -25,7 +25,7 @@ if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/no fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" # run nodeos to generate state files -./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" & +./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" 1>/dev/null 2>/dev/null & sleep 10 kill $! # kill nodeos gracefully, by PID if [[ ! -f data/state/shared_memory.bin ]]; then From ec7f8160c2f9fa07bc23ac1551e92713d477dbff Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:29:22 -0400 Subject: [PATCH 16/73] Add space after test result --- tests/release-build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/release-build.sh b/tests/release-build.sh index 06dc93d4189..bb0d651cc55 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -15,6 +15,7 @@ echo '' [[ -z "$EOSIO_ROOT" ]] && export EOSIO_ROOT="$(echo $(pwd)/ | grep -ioe '.*/eos/' -e '.*/eosio/' -e '.*/build/' | sed 's,/build/,/,')" if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/nodeos/nodeos" ]]; then echo 'ERROR: nodeos binary not found!' + echo '' echo 'Looked in the following places:' echo "$ ls -la \"$EOSIO_ROOT/build/bin\"" ls -la "$EOSIO_ROOT/build/bin" @@ -30,6 +31,7 @@ sleep 10 kill $! # kill nodeos gracefully, by PID if [[ ! -f data/state/shared_memory.bin ]]; then echo 'ERROR: nodeos state not found!' + echo '' echo 'Looked for shared_memory.bin in the following places:' echo "$ ls -la \"$(pwd)/data/state\"" ls -la "$(pwd)/data/state" @@ -41,11 +43,13 @@ fi export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then echo 'PASS: Debug byte not set.' + echo '' rm -rf config data exit 0 fi echo 'FAIL: Debug byte is set!' echo "Debug Byte = 0x$DEBUG_BYTE" +echo '' echo 'First kilobyte of shared_memory.bin:' echo '$ xxd -l 1024 shared_memory.bin' xxd -l 1024 data/state/shared_memory.bin From 2f0311fbf4c310da8acfa7e511fb63a5796567d2 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 12:54:31 -0400 Subject: [PATCH 17/73] Print useful error message if hex dump tool is not found --- tests/release-build.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index bb0d651cc55..945fc583956 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -10,6 +10,13 @@ echo 'performance-critical applications like nodeos. Anyone intending to build a echo 'install nodeos from source should perform a "release build," which excludes' echo 'debugging symbols to generate faster and lighter binaries.' echo '' +# check for xxd +if ! $(xxd --version 2>/dev/null); then + echo 'ERROR: Test requires xxd, but xxd was not found in your PATH!' + echo '' + echo 'The xxd hex dump tool can be installed as part of the vim-common package on most operating systems.' + exit 1 +fi # find nodeos [[ -z "$EOSIO_ROOT" && $(git --version) ]] && export EOSIO_ROOT="$(git rev-parse --show-toplevel)" [[ -z "$EOSIO_ROOT" ]] && export EOSIO_ROOT="$(echo $(pwd)/ | grep -ioe '.*/eos/' -e '.*/eosio/' -e '.*/build/' | sed 's,/build/,/,')" @@ -22,7 +29,7 @@ if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/no echo "$ ls -la \"$EOSIO_ROOT/build/programs/nodeos\"" ls -la "$EOSIO_ROOT/build/programs/nodeos" echo 'Release build test not run.' - exit 1 + exit 2 fi [[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" # run nodeos to generate state files @@ -37,7 +44,7 @@ if [[ ! -f data/state/shared_memory.bin ]]; then ls -la "$(pwd)/data/state" echo 'Release build test not run.' rm -rf config data - exit 2 + exit 3 fi # test state files for debug flag export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" @@ -54,4 +61,4 @@ echo 'First kilobyte of shared_memory.bin:' echo '$ xxd -l 1024 shared_memory.bin' xxd -l 1024 data/state/shared_memory.bin rm -rf config data -exit 3 \ No newline at end of file +exit 4 \ No newline at end of file From 367921f91be1f82d1a2f426fad2a83cf68e9e135 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 01:37:51 -0400 Subject: [PATCH 18/73] Add release-build-test to non-parallelizable test group --- tests/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 70e398cf0cb..519852d370c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -48,6 +48,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/validate-dirty-db.py ${CMAKE_CURRENT_ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/launcher_test.py ${CMAKE_CURRENT_BINARY_DIR}/launcher_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/db_modes_test.sh ${CMAKE_CURRENT_BINARY_DIR}/db_modes_test.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/prod_preactivation_test.py ${CMAKE_CURRENT_BINARY_DIR}/prod_preactivation_test.py COPYONLY) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/release-build.sh ${CMAKE_CURRENT_BINARY_DIR}/release-build.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version-label.sh ${CMAKE_CURRENT_BINARY_DIR}/version-label.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_producer_watermark_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_producer_watermark_test.py COPYONLY) @@ -86,6 +87,8 @@ add_test(NAME launcher_test COMMAND tests/launcher_test.py -v --clean-run --dump set_property(TEST launcher_test PROPERTY LABELS nonparallelizable_tests) add_test(NAME db_modes_test COMMAND tests/db_modes_test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_tests_properties(db_modes_test PROPERTIES COST 6000) +add_test(NAME release-build-test COMMAND tests/release-build.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST release-build-test PROPERTY LABELS nonparallelizable_tests) add_test(NAME version-label-test COMMAND tests/version-label.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) # Long running tests From 3fac1d121663ca5f02a2361356533c14f7b255fb Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 12:36:36 -0400 Subject: [PATCH 19/73] Fix line endings in Ubuntu dockerfiles --- .cicd/platforms/ubuntu-16.04.dockerfile | 3 ++- .cicd/platforms/ubuntu-18.04-unpinned.dockerfile | 3 ++- .cicd/platforms/ubuntu-18.04.dockerfile | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.cicd/platforms/ubuntu-16.04.dockerfile b/.cicd/platforms/ubuntu-16.04.dockerfile index 6bfbe1b1769..20200c889c1 100644 --- a/.cicd/platforms/ubuntu-16.04.dockerfile +++ b/.cicd/platforms/ubuntu-16.04.dockerfile @@ -1,7 +1,8 @@ FROM ubuntu:16.04 ENV VERSION 1 # install dependencies. -RUN apt-get update && apt-get upgrade -y && \ +RUN apt-get update && \ + apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ diff --git a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile index fe682400ea9..c2399822cd3 100644 --- a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile @@ -1,7 +1,8 @@ FROM ubuntu:18.04 ENV VERSION 1 # install dependencies. -RUN apt-get update && apt-get upgrade -y && \ +RUN apt-get update && \ + apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ diff --git a/.cicd/platforms/ubuntu-18.04.dockerfile b/.cicd/platforms/ubuntu-18.04.dockerfile index 02544dfa051..c8a550186af 100644 --- a/.cicd/platforms/ubuntu-18.04.dockerfile +++ b/.cicd/platforms/ubuntu-18.04.dockerfile @@ -1,7 +1,8 @@ FROM ubuntu:18.04 ENV VERSION 1 # install dependencies. -RUN apt-get update && apt-get upgrade -y && \ +RUN apt-get update && \ + apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y git make \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ From 14ab91b2415281fde99db5d67202c848b734fe2d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 12:40:30 -0400 Subject: [PATCH 20/73] Install vim-common on base images to get xxd for hex dumps --- .cicd/platforms/amazon_linux-2-unpinned.dockerfile | 2 +- .cicd/platforms/amazon_linux-2.dockerfile | 2 +- .cicd/platforms/centos-7.6-unpinned.dockerfile | 2 +- .cicd/platforms/centos-7.6.dockerfile | 2 +- .cicd/platforms/ubuntu-16.04.dockerfile | 2 +- .cicd/platforms/ubuntu-18.04-unpinned.dockerfile | 2 +- .cicd/platforms/ubuntu-18.04.dockerfile | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.cicd/platforms/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/amazon_linux-2-unpinned.dockerfile index 4c6d1e3db36..111f808da56 100644 --- a/.cicd/platforms/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/amazon_linux-2-unpinned.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz clang patch + graphviz clang patch vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/amazon_linux-2.dockerfile b/.cicd/platforms/amazon_linux-2.dockerfile index 3f930fcb4de..b63b23c1b02 100644 --- a/.cicd/platforms/amazon_linux-2.dockerfile +++ b/.cicd/platforms/amazon_linux-2.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz patch gcc gcc-c++ + graphviz patch gcc gcc-c++ vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/centos-7.6-unpinned.dockerfile b/.cicd/platforms/centos-7.6-unpinned.dockerfile index cb6bdbb369c..f98047eddd8 100644 --- a/.cicd/platforms/centos-7.6-unpinned.dockerfile +++ b/.cicd/platforms/centos-7.6-unpinned.dockerfile @@ -7,7 +7,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 gettext-devel file libusbx-devel \ - libcurl-devel patch + libcurl-devel patch vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ source /opt/rh/devtoolset-8/enable && \ diff --git a/.cicd/platforms/centos-7.6.dockerfile b/.cicd/platforms/centos-7.6.dockerfile index 9dc0fc25265..ffaecd3c376 100644 --- a/.cicd/platforms/centos-7.6.dockerfile +++ b/.cicd/platforms/centos-7.6.dockerfile @@ -7,7 +7,7 @@ RUN yum update -y && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 gettext-devel file libusbx-devel \ - libcurl-devel patch + libcurl-devel patch vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ source /opt/rh/devtoolset-8/enable && \ diff --git a/.cicd/platforms/ubuntu-16.04.dockerfile b/.cicd/platforms/ubuntu-16.04.dockerfile index 20200c889c1..7dac371e6c1 100644 --- a/.cicd/platforms/ubuntu-16.04.dockerfile +++ b/.cicd/platforms/ubuntu-16.04.dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ - sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https + sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile index c2399822cd3..31cbd4409ed 100644 --- a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache + libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/ubuntu-18.04.dockerfile b/.cicd/platforms/ubuntu-18.04.dockerfile index c8a550186af..bf33c4e906c 100644 --- a/.cicd/platforms/ubuntu-18.04.dockerfile +++ b/.cicd/platforms/ubuntu-18.04.dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch ccache + libcurl4-gnutls-dev pkg-config patch ccache vim-common # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ From 8dbb778318ffaeb74c42638cf408967f52280c3c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 13:44:16 -0400 Subject: [PATCH 21/73] Install jq on base-images for parsing nodeos output --- .cicd/platforms/amazon_linux-2-unpinned.dockerfile | 2 +- .cicd/platforms/amazon_linux-2.dockerfile | 2 +- .cicd/platforms/centos-7.6-unpinned.dockerfile | 3 ++- .cicd/platforms/centos-7.6.dockerfile | 3 ++- .cicd/platforms/ubuntu-16.04.dockerfile | 2 +- .cicd/platforms/ubuntu-18.04-unpinned.dockerfile | 2 +- .cicd/platforms/ubuntu-18.04.dockerfile | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.cicd/platforms/amazon_linux-2-unpinned.dockerfile b/.cicd/platforms/amazon_linux-2-unpinned.dockerfile index 111f808da56..b2c086a6915 100644 --- a/.cicd/platforms/amazon_linux-2-unpinned.dockerfile +++ b/.cicd/platforms/amazon_linux-2-unpinned.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz clang patch vim-common + graphviz clang patch vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/amazon_linux-2.dockerfile b/.cicd/platforms/amazon_linux-2.dockerfile index b63b23c1b02..60afc79a567 100644 --- a/.cicd/platforms/amazon_linux-2.dockerfile +++ b/.cicd/platforms/amazon_linux-2.dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ yum install -y which git sudo procps-ng util-linux autoconf automake \ libtool make bzip2 bzip2-devel openssl-devel gmp-devel libstdc++ libcurl-devel \ libusbx-devel python3 python3-devel python-devel libedit-devel doxygen \ - graphviz patch gcc gcc-c++ vim-common + graphviz patch gcc gcc-c++ vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/centos-7.6-unpinned.dockerfile b/.cicd/platforms/centos-7.6-unpinned.dockerfile index f98047eddd8..789873f824d 100644 --- a/.cicd/platforms/centos-7.6-unpinned.dockerfile +++ b/.cicd/platforms/centos-7.6-unpinned.dockerfile @@ -2,12 +2,13 @@ FROM centos:7.6.1810 ENV VERSION 1 # install dependencies. RUN yum update -y && \ + yum install -y epel-release && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 gettext-devel file libusbx-devel \ - libcurl-devel patch vim-common + libcurl-devel patch vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ source /opt/rh/devtoolset-8/enable && \ diff --git a/.cicd/platforms/centos-7.6.dockerfile b/.cicd/platforms/centos-7.6.dockerfile index ffaecd3c376..552748db63e 100644 --- a/.cicd/platforms/centos-7.6.dockerfile +++ b/.cicd/platforms/centos-7.6.dockerfile @@ -2,12 +2,13 @@ FROM centos:7.6.1810 ENV VERSION 1 # install dependencies. RUN yum update -y && \ + yum install -y epel-release && \ yum --enablerepo=extras install -y centos-release-scl && \ yum --enablerepo=extras install -y devtoolset-8 && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 doxygen \ graphviz bzip2-devel openssl-devel gmp-devel ocaml libicu-devel \ python python-devel rh-python36 gettext-devel file libusbx-devel \ - libcurl-devel patch vim-common + libcurl-devel patch vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ source /opt/rh/devtoolset-8/enable && \ diff --git a/.cicd/platforms/ubuntu-16.04.dockerfile b/.cicd/platforms/ubuntu-16.04.dockerfile index 7dac371e6c1..8cdbe42e9ef 100644 --- a/.cicd/platforms/ubuntu-16.04.dockerfile +++ b/.cicd/platforms/ubuntu-16.04.dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git automake \ libbz2-dev libssl-dev doxygen graphviz libgmp3-dev autotools-dev libicu-dev \ python2.7 python2.7-dev python3 python3-dev autoconf libtool curl zlib1g-dev \ - sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common + sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config apt-transport-https vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile index 31cbd4409ed..4e0feaa0180 100644 --- a/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile +++ b/.cicd/platforms/ubuntu-18.04-unpinned.dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache vim-common + libcurl4-gnutls-dev pkg-config patch llvm-4.0 clang ccache vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ diff --git a/.cicd/platforms/ubuntu-18.04.dockerfile b/.cicd/platforms/ubuntu-18.04.dockerfile index bf33c4e906c..5448a23c04c 100644 --- a/.cicd/platforms/ubuntu-18.04.dockerfile +++ b/.cicd/platforms/ubuntu-18.04.dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ bzip2 automake libbz2-dev libssl-dev doxygen graphviz libgmp3-dev \ autotools-dev libicu-dev python2.7 python2.7-dev python3 python3-dev \ autoconf libtool g++ gcc curl zlib1g-dev sudo ruby libusb-1.0-0-dev \ - libcurl4-gnutls-dev pkg-config patch ccache vim-common + libcurl4-gnutls-dev pkg-config patch ccache vim-common jq # build cmake. RUN curl -LO https://cmake.org/files/v3.13/cmake-3.13.2.tar.gz && \ tar -xzf cmake-3.13.2.tar.gz && \ From 3da14f5f37c562688bb8d24273df52ca8a0bdcba Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 15:48:42 -0400 Subject: [PATCH 22/73] Install jq on Anka base-image, too --- .cicd/platforms/macos-10.14.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/platforms/macos-10.14.sh b/.cicd/platforms/macos-10.14.sh index be31bdbe1ab..6b95666ffbd 100755 --- a/.cicd/platforms/macos-10.14.sh +++ b/.cicd/platforms/macos-10.14.sh @@ -2,7 +2,7 @@ set -eo pipefail VERSION=1 brew update -brew install git cmake python@2 python libtool libusb graphviz automake wget gmp llvm@4 pkgconfig doxygen openssl || true +brew install git cmake python@2 python libtool libusb graphviz automake wget gmp llvm@4 pkgconfig doxygen openssl jq || : if [[ ! $PINNED == false || $UNPINNED == true ]]; then # install clang from source git clone --single-branch --branch release_80 https://git.llvm.org/git/llvm.git clang8 From 31726463e9eca7da657602855e967dc8e12d660c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 17:20:46 -0400 Subject: [PATCH 23/73] Update test purpose, description, and outputs with swatanabe-b1's feedback --- tests/release-build.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index 945fc583956..aef44f15ab7 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -1,14 +1,14 @@ #!/bin/bash # test name and purpose echo '' -echo ' ##### Release Build Test #####' +echo ' ##### Release Build Test #####' echo '' -echo 'The purpose of this test is to ensure that nodeos was built without debugging' -echo 'symbols. Debugging symbols enable software engineers to inspect and control a' -echo 'running program with a debugging tool, but they significantly slow down' -echo 'performance-critical applications like nodeos. Anyone intending to build and' -echo 'install nodeos from source should perform a "release build," which excludes' -echo 'debugging symbols to generate faster and lighter binaries.' +echo ' The purpose of this test is to ensure that nodeos was built with compiler' +echo 'optimizations enabled. While there is no way to programmatically determine that' +echo 'given one binary, we do set a debug flag in nodeos when it is built with' +echo 'asserts. This test checks that debug flag. Anyone intending to build and install' +echo 'nodeos from source should perform a "release build" which excludes asserts and' +echo 'debugging symbols, and performs compiler optimizations.' echo '' # check for xxd if ! $(xxd --version 2>/dev/null); then @@ -49,12 +49,12 @@ fi # test state files for debug flag export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then - echo 'PASS: Debug byte not set.' + echo 'PASS: Debug flag is not set.' echo '' rm -rf config data exit 0 fi -echo 'FAIL: Debug byte is set!' +echo 'FAIL: Debug flag is set!' echo "Debug Byte = 0x$DEBUG_BYTE" echo '' echo 'First kilobyte of shared_memory.bin:' From a97e6dca468bdb662da9be52ad235bdc3e1c0427 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 12 Sep 2019 10:32:51 -0400 Subject: [PATCH 24/73] Use relative paths instead of absolute paths with EOSIO_ROOT --- tests/release-build.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index aef44f15ab7..cccaa1b9d63 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -18,20 +18,18 @@ if ! $(xxd --version 2>/dev/null); then exit 1 fi # find nodeos -[[ -z "$EOSIO_ROOT" && $(git --version) ]] && export EOSIO_ROOT="$(git rev-parse --show-toplevel)" -[[ -z "$EOSIO_ROOT" ]] && export EOSIO_ROOT="$(echo $(pwd)/ | grep -ioe '.*/eos/' -e '.*/eosio/' -e '.*/build/' | sed 's,/build/,/,')" -if [[ ! -f "$EOSIO_ROOT/build/bin/nodeos" && ! -f "$EOSIO_ROOT/build/programs/nodeos/nodeos" ]]; then +[[ $(git --version) ]] && cd "$(git rev-parse --show-toplevel)/build/programs/nodeos" || cd "$(dirname "${BASH_SOURCE[0]}")/../programs/nodeos" +if [[ ! -f nodeos ]]; then echo 'ERROR: nodeos binary not found!' echo '' echo 'Looked in the following places:' - echo "$ ls -la \"$EOSIO_ROOT/build/bin\"" - ls -la "$EOSIO_ROOT/build/bin" - echo "$ ls -la \"$EOSIO_ROOT/build/programs/nodeos\"" - ls -la "$EOSIO_ROOT/build/programs/nodeos" + echo '$ ls -la "$(git rev-parse --show-toplevel)/build/programs/nodeos"' + ls -la "$(git rev-parse --show-toplevel)/build/programs/nodeos" + echo '$ ls -la "$(dirname "${BASH_SOURCE[0]}")/../programs/nodeos"' + ls -la "$(dirname "${BASH_SOURCE[0]}")/../programs/nodeos" echo 'Release build test not run.' exit 2 fi -[[ -f "$EOSIO_ROOT/build/bin/nodeos" ]] && cd "$EOSIO_ROOT/build/bin" || cd "$EOSIO_ROOT/build/programs/nodeos" # run nodeos to generate state files ./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" 1>/dev/null 2>/dev/null & sleep 10 From 76de7c1e403228dc5554a689354b8a8a3da126f7 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 12 Sep 2019 10:57:20 -0400 Subject: [PATCH 25/73] Run release-build-test in isolated subdirectory of /build --- tests/release-build.sh | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/release-build.sh b/tests/release-build.sh index cccaa1b9d63..abcfaa0ecae 100755 --- a/tests/release-build.sh +++ b/tests/release-build.sh @@ -18,38 +18,42 @@ if ! $(xxd --version 2>/dev/null); then exit 1 fi # find nodeos -[[ $(git --version) ]] && cd "$(git rev-parse --show-toplevel)/build/programs/nodeos" || cd "$(dirname "${BASH_SOURCE[0]}")/../programs/nodeos" -if [[ ! -f nodeos ]]; then +[[ $(git --version) ]] && cd "$(git rev-parse --show-toplevel)/build" || cd "$(dirname "${BASH_SOURCE[0]}")/.." +if [[ ! -f programs/nodeos/nodeos ]]; then echo 'ERROR: nodeos binary not found!' echo '' - echo 'Looked in the following places:' - echo '$ ls -la "$(git rev-parse --show-toplevel)/build/programs/nodeos"' - ls -la "$(git rev-parse --show-toplevel)/build/programs/nodeos" - echo '$ ls -la "$(dirname "${BASH_SOURCE[0]}")/../programs/nodeos"' - ls -la "$(dirname "${BASH_SOURCE[0]}")/../programs/nodeos" + echo 'I looked here...' + echo "$ ls -la \"$(pwd)/programs/nodeos\"" + ls -la "$(pwd)/programs/nodeos" + echo '...which I derived from one of these paths:' + echo '$ echo "$(git rev-parse --show-toplevel)/build"' + echo "$(git rev-parse --show-toplevel)/build" + echo '$ echo "$(dirname "${BASH_SOURCE[0]}")/.."' + echo "$(dirname "${BASH_SOURCE[0]}")/.." echo 'Release build test not run.' exit 2 fi # run nodeos to generate state files -./nodeos --config-dir "$(pwd)/config" --data-dir "$(pwd)/data" 1>/dev/null 2>/dev/null & +mkdir release-build-test +programs/nodeos/nodeos --config-dir "$(pwd)/release-build-test/config" --data-dir "$(pwd)/release-build-test/data" 1>/dev/null 2>/dev/null & sleep 10 kill $! # kill nodeos gracefully, by PID -if [[ ! -f data/state/shared_memory.bin ]]; then +if [[ ! -f release-build-test/data/state/shared_memory.bin ]]; then echo 'ERROR: nodeos state not found!' echo '' echo 'Looked for shared_memory.bin in the following places:' - echo "$ ls -la \"$(pwd)/data/state\"" - ls -la "$(pwd)/data/state" + echo "$ ls -la \"$(pwd)/release-build-test/data/state\"" + ls -la "$(pwd)/release-build-test/data/state" echo 'Release build test not run.' - rm -rf config data + rm -rf release-build-test exit 3 fi # test state files for debug flag -export DEBUG_BYTE="$(xxd -seek 9 -l 1 data/state/shared_memory.bin | awk '{print $2}')" +export DEBUG_BYTE="$(xxd -seek 9 -l 1 release-build-test/data/state/shared_memory.bin | awk '{print $2}')" if [[ "$DEBUG_BYTE" == '00' ]]; then echo 'PASS: Debug flag is not set.' echo '' - rm -rf config data + rm -rf release-build-test exit 0 fi echo 'FAIL: Debug flag is set!' @@ -57,6 +61,6 @@ echo "Debug Byte = 0x$DEBUG_BYTE" echo '' echo 'First kilobyte of shared_memory.bin:' echo '$ xxd -l 1024 shared_memory.bin' -xxd -l 1024 data/state/shared_memory.bin -rm -rf config data +xxd -l 1024 release-build-test/data/state/shared_memory.bin +rm -rf release-build-test exit 4 \ No newline at end of file From 5270c95b102c7ed5bd4f52208bc82eb3a75d2e12 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 10 Sep 2019 16:43:28 -0400 Subject: [PATCH 26/73] Simplify comments --- .cicd/generate-pipeline.sh | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index d33a0e681d6..b7025ec43bc 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -1,15 +1,16 @@ #!/bin/bash set -eo pipefail +# environment . ./.cicd/helpers/general.sh export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'} -# Use files in platforms dir as source of truth for what platforms we need to generate steps for export PLATFORMS_JSON_ARRAY='[]' +# read .cicd/platforms for FILE in $(ls $CICD_DIR/platforms); do - # Ability to skip mac or linux by not even creating the json block + # skip mac or linux by not even creating the json block ( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue ( [[ $SKIP_LINUX == true ]] && [[ ! $FILE =~ 'macos' ]] ) && continue - # Prevent using both platform files (only use unpinned or pinned) + # use pinned or unpinned, not both sets of platform files if [[ $PINNED == false || $UNPINNED == true ]] && [[ ! $FILE =~ 'macos' ]]; then export SKIP_CONTRACT_BUILDER=${SKIP_CONTRACT_BUILDER:-true} export SKIP_PACKAGE_BUILDER=${SKIP_PACKAGE_BUILDER:-true} @@ -48,9 +49,8 @@ for FILE in $(ls $CICD_DIR/platforms); do done oIFS="$IFS" IFS=$'' -nIFS=$IFS # Needed to fix array splitting (\n won't work) -################### -# Anka Ensure Tag # +nIFS=$IFS # fix array splitting (\n won't work) +# base-image steps echo $PLATFORMS_JSON_ARRAY | jq -cr ".[]" | while read -r PLATFORM_JSON; do if [[ $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) =~ 'macos' ]]; then cat < Date: Tue, 10 Sep 2019 16:55:00 -0400 Subject: [PATCH 27/73] Add $RUNS for loop --- .cicd/generate-pipeline.sh | 238 +++++++++++++++++++------------------ 1 file changed, 121 insertions(+), 117 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index b7025ec43bc..c1d95721168 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -5,6 +5,7 @@ set -eo pipefail export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'} export PLATFORMS_JSON_ARRAY='[]' +[[ -z "$RUNS" ]] && export RUNS='1' # read .cicd/platforms for FILE in $(ls $CICD_DIR/platforms); do # skip mac or linux by not even creating the json block @@ -120,137 +121,140 @@ EOF fi done echo " - wait"; echo "" -# parallel tests -echo $PLATFORMS_JSON_ARRAY | jq -cr ".[]" | while read -r PLATFORM_JSON; do - if [[ ! $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) =~ 'macos' ]]; then - cat < Date: Tue, 10 Sep 2019 16:57:50 -0400 Subject: [PATCH 28/73] Add wait step between test batches --- .cicd/generate-pipeline.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index c1d95721168..a373dcc3d10 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -255,6 +255,10 @@ EOF done IFS=$nIFS done + if [[ "$RUN" != "$RUNS" ]]; then + echo ' - wait' + echo '' + fi done # pipeline tail cat < Date: Tue, 10 Sep 2019 17:04:19 -0400 Subject: [PATCH 29/73] Clearer strings --- .cicd/generate-pipeline.sh | 44 ++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index a373dcc3d10..cbad3ffcfea 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -19,14 +19,14 @@ for FILE in $(ls $CICD_DIR/platforms); do else [[ $FILE =~ 'unpinned' ]] && continue fi - export FILE_NAME=$(echo $FILE | awk '{split($0,a,/\.(d|s)/); print a[1] }') - export PLATFORM_NAME=$(echo $FILE_NAME | cut -d- -f1 | sed 's/os/OS/g') - export PLATFORM_NAME_UPCASE=$(echo $PLATFORM_NAME | tr a-z A-Z) - export VERSION_MAJOR=$(echo $FILE_NAME | cut -d- -f2 | cut -d. -f1) - [[ $(echo $FILE_NAME | cut -d- -f2) =~ '.' ]] && export VERSION_MINOR="_$(echo $FILE_NAME | cut -d- -f2 | cut -d. -f2)" || export VERSION_MINOR='' - export VERSION_FULL=$(echo $FILE_NAME | cut -d- -f2) + export FILE_NAME="$(echo $FILE | awk '{split($0,a,/\.(d|s)/); print a[1] }')" + export PLATFORM_NAME="$(echo $FILE_NAME | cut -d- -f1 | sed 's/os/OS/g')" + export PLATFORM_NAME_UPCASE="$(echo $PLATFORM_NAME | tr a-z A-Z)" + export VERSION_MAJOR="$(echo $FILE_NAME | cut -d- -f2 | cut -d. -f1)" + [[ "$(echo $FILE_NAME | cut -d- -f2)" =~ '.' ]] && export VERSION_MINOR="_$(echo $FILE_NAME | cut -d- -f2 | cut -d. -f2)" || export VERSION_MINOR='' + export VERSION_FULL="$(echo $FILE_NAME | cut -d- -f2)" OLDIFS=$IFS - IFS="_" + IFS='_' set $PLATFORM_NAME IFS=$OLDIFS export PLATFORM_NAME_FULL="$(capitalize $1)$( [[ ! -z $2 ]] && echo "_$(capitalize $2)" || true ) $VERSION_FULL" @@ -52,8 +52,8 @@ oIFS="$IFS" IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) # base-image steps -echo $PLATFORMS_JSON_ARRAY | jq -cr ".[]" | while read -r PLATFORM_JSON; do - if [[ $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) =~ 'macos' ]]; then +echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do + if [[ "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Tue, 10 Sep 2019 17:05:42 -0400 Subject: [PATCH 30/73] Remove extra space --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index cbad3ffcfea..d1dd810ea71 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -10,7 +10,7 @@ export PLATFORMS_JSON_ARRAY='[]' for FILE in $(ls $CICD_DIR/platforms); do # skip mac or linux by not even creating the json block ( [[ $SKIP_MAC == true ]] && [[ $FILE =~ 'macos' ]] ) && continue - ( [[ $SKIP_LINUX == true ]] && [[ ! $FILE =~ 'macos' ]] ) && continue + ( [[ $SKIP_LINUX == true ]] && [[ ! $FILE =~ 'macos' ]] ) && continue # use pinned or unpinned, not both sets of platform files if [[ $PINNED == false || $UNPINNED == true ]] && [[ ! $FILE =~ 'macos' ]]; then export SKIP_CONTRACT_BUILDER=${SKIP_CONTRACT_BUILDER:-true} From 7222f282da516cbe958f5f8ce319ae4939f6d779 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 14:42:57 -0400 Subject: [PATCH 31/73] Fix YAML whitespace --- .cicd/generate-pipeline.sh | 177 ++++++++++++++++++------------------- 1 file changed, 86 insertions(+), 91 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index d1dd810ea71..1b46fceec27 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -59,8 +59,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do command: - "git clone git@github.com:EOSIO/mac-anka-fleet.git" - "cd mac-anka-fleet && . ./ensure_tag.bash -u 12 -r 25G -a '-n'" - agents: - - "queue=mac-anka-templater-fleet" + agents: "queue=mac-anka-templater-fleet" env: REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} REPO_COMMIT: $BUILDKITE_COMMIT @@ -114,8 +113,7 @@ EOF debug: true wait-network: true timeout: ${TIMEOUT:-180} - agents: - - "queue=mac-anka-large-node-fleet" + agents: "queue=mac-anka-large-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD} EOF @@ -129,39 +127,38 @@ for RUN in $(seq 1 $RUNS); do echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Wed, 11 Sep 2019 14:57:21 -0400 Subject: [PATCH 32/73] Fix YAML spacing, round 2 --- .cicd/generate-pipeline.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 1b46fceec27..5957fb5cf92 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -132,8 +132,8 @@ for RUN in $(seq 1 $RUNS); do - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "./.cicd/test.sh scripts/parallel-test.sh" env: - IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) - BUILDKITE_AGENT_ACCESS_TOKEN: + IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) + BUILDKITE_AGENT_ACCESS_TOKEN: agents: queue: "automation-eos-builder-fleet" timeout: ${TIMEOUT:-10} @@ -175,8 +175,8 @@ EOF - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "./.cicd/test.sh scripts/serial-test.sh $TEST_NAME" env: - IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) - BUILDKITE_AGENT_ACCESS_TOKEN: + IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) + BUILDKITE_AGENT_ACCESS_TOKEN: agents: queue: "automation-eos-builder-fleet" timeout: ${TIMEOUT:-20} @@ -219,8 +219,8 @@ EOF - "buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "./.cicd/test.sh scripts/long-running-test.sh $TEST_NAME" env: - IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) - BUILDKITE_AGENT_ACCESS_TOKEN: + IMAGE_TAG: $(echo "$PLATFORM_JSON" | jq -r .FILE_NAME) + BUILDKITE_AGENT_ACCESS_TOKEN: agents: queue: "automation-eos-builder-fleet" timeout: ${TIMEOUT:-180} From c6459ef641f53a6dd35b477ede48c399898a8c48 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 15:35:41 -0400 Subject: [PATCH 33/73] Fix misaligned cat --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 5957fb5cf92..4d842f79ad2 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -54,7 +54,7 @@ nIFS=$IFS # fix array splitting (\n won't work) # base-image steps echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then - cat < Date: Wed, 11 Sep 2019 15:37:45 -0400 Subject: [PATCH 34/73] Put wait step at top of YAML --- .cicd/generate-pipeline.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 4d842f79ad2..8ff6e44e47e 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -51,6 +51,9 @@ done oIFS="$IFS" IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) +# start with a wait step +echo ' - wait' +echo '' # base-image steps echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then @@ -72,11 +75,11 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do timeout: ${TIMEOUT:-320} skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_ENSURE_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)} + - wait + EOF fi done -echo ' - wait' -echo '' # build steps echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then From 0f0702fcd52a871ec196979d82f88265d861aa4d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 15:40:46 -0400 Subject: [PATCH 35/73] Add space after macOS serial tests in YAML --- .cicd/generate-pipeline.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 8ff6e44e47e..c74183c941b 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -205,6 +205,7 @@ EOF timeout: ${TIMEOUT:-20} agents: "queue=mac-anka-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS} + EOF fi done From a5397df9fcdef8943d879190f7a299a60e33b2e2 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 15:44:28 -0400 Subject: [PATCH 36/73] Use comments to break up blocks of YAML, to help with debugging --- .cicd/generate-pipeline.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index c74183c941b..05ebc33174f 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -55,6 +55,7 @@ nIFS=$IFS # fix array splitting (\n won't work) echo ' - wait' echo '' # base-image steps +echo ' # base-images' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Wed, 11 Sep 2019 15:50:04 -0400 Subject: [PATCH 37/73] Print run group index in YAML comment --- .cicd/generate-pipeline.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 05ebc33174f..0a2e69fd957 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -128,6 +128,7 @@ echo ' - wait' echo '' # tests for RUN in $(seq 1 $RUNS); do + echo " # run group $RUN of $RUNS" # parallel tests echo ' # parallel tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do From 44cdde6bd306a1759357861c0b6545e7f40889df Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 16:42:24 -0400 Subject: [PATCH 38/73] Fix IFS so FOR loop works --- .cicd/generate-pipeline.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 0a2e69fd957..38b206c80fd 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -49,7 +49,7 @@ for FILE in $(ls $CICD_DIR/platforms); do }]') done oIFS="$IFS" -IFS=$'' +IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) # start with a wait step echo ' - wait' @@ -127,7 +127,9 @@ done echo ' - wait' echo '' # tests +IFS=$oIFS for RUN in $(seq 1 $RUNS); do + IFS=$'' echo " # run group $RUN of $RUNS" # parallel tests echo ' # parallel tests' @@ -262,6 +264,7 @@ EOF done IFS=$nIFS done + IFS=$oIFS if [[ "$RUN" != "$RUNS" ]]; then echo ' - wait' echo '' From 9b48774fe0bc9bc53a533937e99ef3e9f1f00a3c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 11 Sep 2019 16:42:45 -0400 Subject: [PATCH 39/73] Remove extra space --- .cicd/generate-pipeline.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 38b206c80fd..32bda923ef0 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -272,7 +272,6 @@ EOF done # pipeline tail cat < Date: Thu, 12 Sep 2019 15:41:12 -0400 Subject: [PATCH 40/73] Rename RUNS variable to ROUNDS to avoid confusion when writing about multiple test-stability runs --- .cicd/generate-pipeline.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 32bda923ef0..9508cd1e1f5 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -5,7 +5,7 @@ set -eo pipefail export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'} export PLATFORMS_JSON_ARRAY='[]' -[[ -z "$RUNS" ]] && export RUNS='1' +[[ -z "$ROUNDS" ]] && export ROUNDS='1' # read .cicd/platforms for FILE in $(ls $CICD_DIR/platforms); do # skip mac or linux by not even creating the json block @@ -128,9 +128,9 @@ echo ' - wait' echo '' # tests IFS=$oIFS -for RUN in $(seq 1 $RUNS); do +for ROUND in $(seq 1 $ROUNDS); do IFS=$'' - echo " # run group $RUN of $RUNS" + echo " # round $ROUND of $ROUNDS" # parallel tests echo ' # parallel tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do @@ -265,7 +265,7 @@ EOF IFS=$nIFS done IFS=$oIFS - if [[ "$RUN" != "$RUNS" ]]; then + if [[ "$ROUND" != "$ROUNDS" ]]; then echo ' - wait' echo '' fi From f1672b70b86edbaa7e9d6bd5df3e27a5a0482835 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Thu, 12 Sep 2019 16:51:30 -0400 Subject: [PATCH 41/73] Don't wait between base-image steps, wait after them --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 9508cd1e1f5..1e11060236e 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -76,11 +76,11 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do timeout: ${TIMEOUT:-320} skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_ENSURE_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)} - - wait - EOF fi done +echo ' - wait' +echo '' # build steps echo ' # builds' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do From 6d9321e03b67c15515d3664ae8e9cc6435c6422a Mon Sep 17 00:00:00 2001 From: Scott Arnette Date: Fri, 13 Sep 2019 15:37:18 -0400 Subject: [PATCH 42/73] Port PR #7901 to 1.8.x --- .cicd/generate-pipeline.sh | 70 ++++++++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 1e11060236e..560619d0501 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -48,6 +48,15 @@ for FILE in $(ls $CICD_DIR/platforms); do "ICON": env.ICON }]') done +# set build_source whether triggered or not +if [[ ! -z ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} ]]; then + export BUILD_SOURCE="--build \$BUILDKITE_TRIGGERED_FROM_BUILD_ID" +fi +export BUILD_SOURCE=${BUILD_SOURCE:---build \$BUILDKITE_BUILD_ID} +# set trigger_job if master/release/develop branch and webhook +if [[ $BUILDKITE_BRANCH =~ ^release/[0-9]+\.[0-9]+\.x$ || $BUILDKITE_BRANCH =~ ^master$ || $BUILDKITE_BRANCH =~ ^develop$ ]]; then + [[ $BUILDKITE_SOURCE == 'webhook' ]] && export TRIGGER_JOB=true +fi oIFS="$IFS" IFS=$'' nIFS=$IFS # fix array splitting (\n won't work) @@ -96,7 +105,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do agents: queue: "automation-eos-builder-fleet" timeout: ${TIMEOUT:-180} - skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD} + skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_BUILD} EOF else @@ -119,7 +128,7 @@ EOF wait-network: true timeout: ${TIMEOUT:-180} agents: "queue=mac-anka-large-node-fleet" - skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD} + skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_BUILD} EOF fi @@ -146,7 +155,7 @@ for ROUND in $(seq 1 $ROUNDS); do agents: queue: "automation-eos-builder-fleet" timeout: ${TIMEOUT:-10} - skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS} + skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_UNIT_TESTS} EOF else @@ -167,7 +176,7 @@ EOF wait-network: true timeout: ${TIMEOUT:-20} agents: "queue=mac-anka-node-fleet" - skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS} + skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_UNIT_TESTS} EOF fi @@ -190,7 +199,7 @@ EOF agents: queue: "automation-eos-builder-fleet" timeout: ${TIMEOUT:-20} - skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS} + skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_SERIAL_TESTS} EOF else @@ -211,7 +220,7 @@ EOF wait-network: true timeout: ${TIMEOUT:-20} agents: "queue=mac-anka-node-fleet" - skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS} + skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}${SKIP_SERIAL_TESTS} EOF fi @@ -228,7 +237,7 @@ EOF cat < Date: Fri, 13 Sep 2019 16:18:23 -0400 Subject: [PATCH 43/73] anka plugin update and new features for failover of registry --- .cicd/generate-pipeline.sh | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 1e11060236e..1572e6d3b57 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -107,7 +107,7 @@ EOF - "cd eos && ./.cicd/build.sh" - "cd eos && tar -pczf build.tar.gz build && buildkite-agent artifact upload build.tar.gz" plugins: - - chef/anka#v0.5.1: + - chef/anka#v0.5.4: no-volume: true inherit-environment-vars: true vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} @@ -117,6 +117,10 @@ EOF always-pull: true debug: true wait-network: true + failover-registries: + - 'registry_1' + - 'registry_2' + pre-execute-sleep: 5 timeout: ${TIMEOUT:-180} agents: "queue=mac-anka-large-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_BUILD} @@ -157,7 +161,7 @@ EOF - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "cd eos && ./.cicd/test.sh scripts/parallel-test.sh" plugins: - - chef/anka#v0.5.1: + - chef/anka#v0.5.4: no-volume: true inherit-environment-vars: true vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} @@ -165,6 +169,10 @@ EOF always-pull: true debug: true wait-network: true + failover-registries: + - 'registry_1' + - 'registry_2' + pre-execute-sleep: 5 timeout: ${TIMEOUT:-20} agents: "queue=mac-anka-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_UNIT_TESTS} @@ -201,7 +209,7 @@ EOF - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "cd eos && ./.cicd/test.sh scripts/serial-test.sh $TEST_NAME" plugins: - - chef/anka#v0.5.1: + - chef/anka#v0.5.4: no-volume: true inherit-environment-vars: true vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} @@ -209,6 +217,10 @@ EOF always-pull: true debug: true wait-network: true + failover-registries: + - 'registry_1' + - 'registry_2' + pre-execute-sleep: 5 timeout: ${TIMEOUT:-20} agents: "queue=mac-anka-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_SERIAL_TESTS} @@ -247,7 +259,7 @@ EOF - "cd eos && buildkite-agent artifact download build.tar.gz . --step '$(echo "$PLATFORM_JSON" | jq -r .ICON) $(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_FULL) - Build' && tar -xzf build.tar.gz" - "cd eos && ./.cicd/test.sh scripts/long-running-test.sh $TEST_NAME" plugins: - - chef/anka#v0.5.1: + - chef/anka#v0.5.4: no-volume: true inherit-environment-vars: true vm-name: ${MOJAVE_ANKA_TEMPLATE_NAME} @@ -255,6 +267,10 @@ EOF always-pull: true debug: true wait-network: true + failover-registries: + - 'registry_1' + - 'registry_2' + pre-execute-sleep: 5 timeout: ${TIMEOUT:-180} agents: "queue=mac-anka-node-fleet" skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_LONG_RUNNING_TESTS:-true} @@ -337,7 +353,7 @@ cat < Date: Fri, 13 Sep 2019 17:54:37 -0400 Subject: [PATCH 44/73] added fix --- .cicd/generate-pipeline.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 560619d0501..8e0fab290d6 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -5,6 +5,13 @@ set -eo pipefail export MOJAVE_ANKA_TAG_BASE=${MOJAVE_ANKA_TAG_BASE:-'clean::cicd::git-ssh::nas::brew::buildkite-agent'} export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40G'} export PLATFORMS_JSON_ARRAY='[]' + +# Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url +if [[ $BUILDKITE_BRANCH =~ 'pull/' ]]; then + PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) + export GIT_FETCH="git fetch -v --prune origin refs/pull/$PR_ID/head &&" +fi + [[ -z "$ROUNDS" ]] && export ROUNDS='1' # read .cicd/platforms for FILE in $(ls $CICD_DIR/platforms); do @@ -112,7 +119,7 @@ EOF cat < Date: Fri, 13 Sep 2019 17:56:28 -0400 Subject: [PATCH 45/73] added fix --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 8e0fab290d6..c5a7f230177 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -87,7 +87,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do TEMPLATE_TAG: $MOJAVE_ANKA_TAG_BASE PINNED: $PINNED UNPINNED: $UNPINNED - TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eos && cd eos && git checkout $BUILDKITE_COMMIT && git submodule update --init --recursive && export PINNED=$PINNED && export UNPINNED=$UNPINNED && . ./.cicd/platforms/macos-10.14.sh && cd ~/eos && cd .. && rm -rf eos" + TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eos && cd eos && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && export PINNED=$PINNED && export UNPINNED=$UNPINNED && . ./.cicd/platforms/macos-10.14.sh && cd ~/eos && cd .. && rm -rf eos" PROJECT_TAG: $(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) timeout: ${TIMEOUT:-320} skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_ENSURE_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)} From 31f2195001a5a091da1959210184ee1e41841b9e Mon Sep 17 00:00:00 2001 From: Nathan Pierce Date: Fri, 13 Sep 2019 17:57:31 -0400 Subject: [PATCH 46/73] quick fixes --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index c5a7f230177..5ed6c56f6ff 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -87,7 +87,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do TEMPLATE_TAG: $MOJAVE_ANKA_TAG_BASE PINNED: $PINNED UNPINNED: $UNPINNED - TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eos && cd eos && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && export PINNED=$PINNED && export UNPINNED=$UNPINNED && . ./.cicd/platforms/macos-10.14.sh && cd ~/eos && cd .. && rm -rf eos" + TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eos && cd eos && $GIT_FETCH git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && export PINNED=$PINNED && export UNPINNED=$UNPINNED && . ./.cicd/platforms/macos-10.14.sh && cd ~/eos && cd .. && rm -rf eos" PROJECT_TAG: $(echo "$PLATFORM_JSON" | jq -r .HASHED_IMAGE_TAG) timeout: ${TIMEOUT:-320} skip: \${SKIP_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)}\${SKIP_ENSURE_$(echo "$PLATFORM_JSON" | jq -r .PLATFORM_NAME_UPCASE)_$(echo "$PLATFORM_JSON" | jq -r .VERSION_MAJOR)$(echo "$PLATFORM_JSON" | jq -r .VERSION_MINOR)} From 3880183d03c2d88ba1d5ccb2b26798027e31859d Mon Sep 17 00:00:00 2001 From: Nathan Pierce Date: Fri, 13 Sep 2019 18:22:33 -0400 Subject: [PATCH 47/73] quick fix --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 5ed6c56f6ff..5836923f591 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -7,7 +7,7 @@ export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40 export PLATFORMS_JSON_ARRAY='[]' # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url -if [[ $BUILDKITE_BRANCH =~ 'pull/' ]]; then +if [[ $BUILDKITE_BRANCH =~ '^pull/[0-9]+/head:' ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) export GIT_FETCH="git fetch -v --prune origin refs/pull/$PR_ID/head &&" fi From a3a47a16118d4e278f80b20018412451399efd5d Mon Sep 17 00:00:00 2001 From: Nathan Pierce Date: Fri, 13 Sep 2019 18:53:29 -0400 Subject: [PATCH 48/73] removed quote from GIT_FETCH/pull regex --- .cicd/generate-pipeline.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 5836923f591..0cf6ae1e036 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -7,7 +7,7 @@ export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40 export PLATFORMS_JSON_ARRAY='[]' # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url -if [[ $BUILDKITE_BRANCH =~ '^pull/[0-9]+/head:' ]]; then +if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) export GIT_FETCH="git fetch -v --prune origin refs/pull/$PR_ID/head &&" fi @@ -97,6 +97,7 @@ EOF done echo ' - wait' echo '' +exit # build steps echo ' # builds' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do From 652c10a6343af623762a26212b857fdf39f7543d Mon Sep 17 00:00:00 2001 From: Nathan Pierce Date: Fri, 13 Sep 2019 18:54:55 -0400 Subject: [PATCH 49/73] added back : --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 0cf6ae1e036..4d64ee29fdd 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -7,7 +7,7 @@ export MOJAVE_ANKA_TEMPLATE_NAME=${MOJAVE_ANKA_TEMPLATE_NAME:-'10.14.4_6C_14G_40 export PLATFORMS_JSON_ARRAY='[]' # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url -if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head ]]; then +if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) export GIT_FETCH="git fetch -v --prune origin refs/pull/$PR_ID/head &&" fi From 0ef63b25465e5593ea517e3391411eb5a29cd9bc Mon Sep 17 00:00:00 2001 From: Nathan Pierce Date: Fri, 13 Sep 2019 18:57:45 -0400 Subject: [PATCH 50/73] removed exit --- .cicd/generate-pipeline.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 4d64ee29fdd..921599f0fcb 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -97,7 +97,6 @@ EOF done echo ' - wait' echo '' -exit # build steps echo ' # builds' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do From e796f37821919be9e20a4cc32109bee29e5eeb8e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 16 Sep 2019 11:17:15 -0400 Subject: [PATCH 51/73] Fix problem with keepalive timer sometimes causing a crash on exit because plugin_shutdown not called. Moved the creation of keepalive timer to after acceptor bind which can easily fail if port already in use. Changed fc_elog to elog for port already in use so that it is always logged regardless of net_plugin_impl logger setting. Also move the setup of logger to start of plugin_startup since logging is used within the method. --- plugins/net_plugin/net_plugin.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index b35b38a0da8..b563a36294a 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3022,6 +3022,8 @@ namespace eosio { } void net_plugin::plugin_startup() { + handle_sighup(); + try { my->producer_plug = app().find_plugin(); // currently thread_pool only used for server_ioc @@ -3056,17 +3058,13 @@ namespace eosio { } } - my->keepalive_timer.reset( new boost::asio::steady_timer( my->thread_pool->get_executor() ) ); - my->ticker(); - if( my->acceptor ) { my->acceptor->open(my->listen_endpoint.protocol()); my->acceptor->set_option(tcp::acceptor::reuse_address(true)); try { my->acceptor->bind(my->listen_endpoint); } catch (const std::exception& e) { - fc_elog( logger, "net_plugin::plugin_startup failed to bind to port ${port}", - ("port", my->listen_endpoint.port())); + elog( "net_plugin::plugin_startup failed to bind to port ${port}", ("port", my->listen_endpoint.port())); throw e; } my->acceptor->listen(); @@ -3078,6 +3076,9 @@ namespace eosio { cc.accepted_block.connect( boost::bind(&net_plugin_impl::accepted_block, my.get(), _1)); } + my->keepalive_timer.reset( new boost::asio::steady_timer( my->thread_pool->get_executor() ) ); + my->ticker(); + my->incoming_transaction_ack_subscription = app().get_channel().subscribe(boost::bind(&net_plugin_impl::transaction_ack, my.get(), _1)); if( cc.get_read_mode() == chain::db_read_mode::READ_ONLY ) { @@ -3090,7 +3091,12 @@ namespace eosio { for( auto seed_node : my->supplied_peers ) { connect( seed_node ); } - handle_sighup(); + + } catch (...) { + // always want plugin_shutdown even on exception + plugin_shutdown(); + throw; + } } void net_plugin::handle_sighup() { From e122844e64d6f2a3e275095ffbeeb4b7b1ac489d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 16 Sep 2019 11:17:53 -0400 Subject: [PATCH 52/73] Make sure plugin_shutdown is called in case of exception in plugin_startup to ensure proper shutdown on exception --- plugins/producer_plugin/producer_plugin.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 4291dffaeb6..a9cfab68cb4 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -804,6 +804,7 @@ void producer_plugin::plugin_startup() { try { handle_sighup(); // Sets loggers + try { ilog("producer plugin: plugin_startup() begin"); chain::controller& chain = my->chain_plug->chain(); @@ -840,6 +841,11 @@ void producer_plugin::plugin_startup() my->schedule_production_loop(); ilog("producer plugin: plugin_startup() end"); + } catch( ... ) { + // always call plugin_shutdown, even on exception + plugin_shutdown(); + throw; + } } FC_CAPTURE_AND_RETHROW() } void producer_plugin::plugin_shutdown() { From 77f2e58aed75f90145324a086842b7850fabdd8c Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 16 Sep 2019 13:56:01 -0400 Subject: [PATCH 53/73] Fix release build type for macOS on Travis CI --- .cicd/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index 2688b65413a..beb7a3ab8a0 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -5,7 +5,7 @@ mkdir -p $BUILD_DIR CMAKE_EXTRAS="-DBUILD_MONGO_DB_PLUGIN=true -DCMAKE_BUILD_TYPE='Release'" if [[ $(uname) == 'Darwin' ]]; then # You can't use chained commands in execute - [[ $TRAVIS == true ]] && export PINNED=false && ccache -s && CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" && ./$CICD_DIR/platforms/macos-10.14.sh + [[ $TRAVIS == true ]] && export PINNED=false && ccache -s && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" && ./$CICD_DIR/platforms/macos-10.14.sh ( [[ ! $PINNED == false || $UNPINNED == true ]] ) && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" cd $BUILD_DIR cmake $CMAKE_EXTRAS .. From 3352e25abf013e5f161c59bd500cd0de79fd59f3 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 16 Sep 2019 13:58:28 -0400 Subject: [PATCH 54/73] Clean up IF statements --- .cicd/build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index beb7a3ab8a0..e921e4586d4 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -5,8 +5,13 @@ mkdir -p $BUILD_DIR CMAKE_EXTRAS="-DBUILD_MONGO_DB_PLUGIN=true -DCMAKE_BUILD_TYPE='Release'" if [[ $(uname) == 'Darwin' ]]; then # You can't use chained commands in execute - [[ $TRAVIS == true ]] && export PINNED=false && ccache -s && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" && ./$CICD_DIR/platforms/macos-10.14.sh - ( [[ ! $PINNED == false || $UNPINNED == true ]] ) && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" + if [[ $TRAVIS == true ]]; then + export PINNED=false + ccache -s + CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + ./$CICD_DIR/platforms/macos-10.14.sh + fi + [[ ! $PINNED == false || $UNPINNED == true ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" cd $BUILD_DIR cmake $CMAKE_EXTRAS .. make -j$JOBS From 23c4069361259e09e220526dccbec8f0484b52be Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 16 Sep 2019 14:01:41 -0400 Subject: [PATCH 55/73] Quoted strings --- .cicd/build.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index e921e4586d4..21cf5ea2953 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -3,15 +3,15 @@ set -eo pipefail . ./.cicd/helpers/general.sh mkdir -p $BUILD_DIR CMAKE_EXTRAS="-DBUILD_MONGO_DB_PLUGIN=true -DCMAKE_BUILD_TYPE='Release'" -if [[ $(uname) == 'Darwin' ]]; then +if [[ "$(uname)" == 'Darwin' ]]; then # You can't use chained commands in execute - if [[ $TRAVIS == true ]]; then + if [[ "$TRAVIS" == 'true' ]]; then export PINNED=false ccache -s CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" ./$CICD_DIR/platforms/macos-10.14.sh fi - [[ ! $PINNED == false || $UNPINNED == true ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" + [[ ! "$PINNED" == 'false' || "$UNPINNED" == 'true' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" cd $BUILD_DIR cmake $CMAKE_EXTRAS .. make -j$JOBS @@ -21,33 +21,33 @@ else # Linux PRE_COMMANDS="cd $MOUNTED_DIR/build" # PRE_COMMANDS: Executed pre-cmake # CMAKE_EXTRAS: Executed within and right before the cmake path (cmake CMAKE_EXTRAS ..) - [[ ! $IMAGE_TAG =~ 'unpinned' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$MOUNTED_DIR/.cicd/helpers/clang.make -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - if [[ $IMAGE_TAG == 'amazon_linux-2' ]]; then + [[ ! "$IMAGE_TAG" =~ 'unpinned' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$MOUNTED_DIR/.cicd/helpers/clang.make -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + if [[ "$IMAGE_TAG" == 'amazon_linux-2' ]]; then PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH" - elif [[ $IMAGE_TAG == 'centos-7.6' ]]; then + elif [[ "$IMAGE_TAG" == 'centos-7.6' ]]; then PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH" - elif [[ $IMAGE_TAG == 'ubuntu-16.04' ]]; then + elif [[ "$IMAGE_TAG" == 'ubuntu-16.04' ]]; then PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH" - elif [[ $IMAGE_TAG == 'ubuntu-18.04' ]]; then + elif [[ "$IMAGE_TAG" == 'ubuntu-18.04' ]]; then PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH" - elif [[ $IMAGE_TAG == 'amazon_linux-2-unpinned' ]]; then + elif [[ "$IMAGE_TAG" == 'amazon_linux-2-unpinned' ]]; then PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH" CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" - elif [[ $IMAGE_TAG == 'centos-7.6-unpinned' ]]; then + elif [[ "$IMAGE_TAG" == 'centos-7.6-unpinned' ]]; then PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:\\\$PATH" - elif [[ $IMAGE_TAG == 'ubuntu-18.04-unpinned' ]]; then + elif [[ "$IMAGE_TAG" == 'ubuntu-18.04-unpinned' ]]; then PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH" CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" fi BUILD_COMMANDS="cmake $CMAKE_EXTRAS .. && make -j$JOBS" # Docker Commands - if [[ $BUILDKITE == true ]]; then + if [[ "$BUILDKITE" == 'true' ]]; then # Generate Base Images $CICD_DIR/generate-base-images.sh - [[ $ENABLE_INSTALL == true ]] && COMMANDS="cp -r $MOUNTED_DIR /root/eosio && cd /root/eosio/build &&" + [[ "$ENABLE_INSTALL" == 'true' ]] && COMMANDS="cp -r $MOUNTED_DIR /root/eosio && cd /root/eosio/build &&" COMMANDS="$COMMANDS $BUILD_COMMANDS" - [[ $ENABLE_INSTALL == true ]] && COMMANDS="$COMMANDS && make install" - elif [[ $TRAVIS == true ]]; then + [[ "$ENABLE_INSTALL" == 'true' ]] && COMMANDS="$COMMANDS && make install" + elif [[ "$TRAVIS" == 'true' ]]; then ARGS="$ARGS -v /usr/lib/ccache -v $HOME/.ccache:/opt/.ccache -e JOBS -e TRAVIS -e CCACHE_DIR=/opt/.ccache" COMMANDS="ccache -s && $BUILD_COMMANDS" fi From 769cd672b80f8855bb389ea5651c440d62e003b0 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 16 Sep 2019 18:03:01 -0400 Subject: [PATCH 56/73] Refactor producer plugin start_block into a few methods to make the logic easier to follow --- plugins/producer_plugin/producer_plugin.cpp | 486 ++++++++++---------- 1 file changed, 253 insertions(+), 233 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index a9cfab68cb4..a8b4f87a6c8 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -202,8 +202,13 @@ class producer_plugin_impl : public std::enable_shared_from_this(); - auto& persisted_by_expiry = _persistent_transactions.get(); - if (!persisted_by_expiry.empty()) { - int num_expired_persistent = 0; - int orig_count = _persistent_transactions.size(); - - while(!persisted_by_expiry.empty() && persisted_by_expiry.begin()->expiry <= pending_block_time) { - if (preprocess_deadline <= fc::time_point::now()) { - exhausted = true; - break; - } - auto const& txid = persisted_by_expiry.begin()->trx_id; - if (_pending_block_mode == pending_block_mode::producing) { - fc_dlog(_trx_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is EXPIRING PERSISTED tx: ${txid}", - ("block_num", chain.head_block_num() + 1) - ("prod", chain.pending_block_producer()) - ("txid", txid)); - } else { - fc_dlog(_trx_trace_log, "[TRX_TRACE] Speculative execution is EXPIRING PERSISTED tx: ${txid}", - ("txid", txid)); - } + remove_expired_persisted_trxs( preprocess_deadline ); + remove_expired_blacklisted_trxs( preprocess_deadline ); + + try { + size_t orig_pending_txn_size = _pending_incoming_transactions.size(); + + // attempt to play persisted transactions first + process_unapplied_trxs( preprocess_deadline ); - persisted_by_expiry.erase(persisted_by_expiry.begin()); - num_expired_persistent++; + if (_pending_block_mode == pending_block_mode::producing) { + process_scheduled_and_incoming_trxs( preprocess_deadline, orig_pending_txn_size ); } - if( exhausted ) { - fc_wlog( _log, "Unable to process all ${n} persisted transactions before deadline, Expired ${expired}", - ( "n", orig_count ) - ( "expired", num_expired_persistent ) ); + if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit() + return start_block_result::failed; + if( preprocess_deadline <= fc::time_point::now() ) { + return start_block_result::exhausted; } else { - fc_dlog( _log, "Processed ${n} persisted transactions, Expired ${expired}", - ( "n", orig_count ) - ( "expired", num_expired_persistent ) ); + _incoming_trx_weight = 0.0; + + // attempt to apply any pending incoming transactions + process_incoming_trxs( preprocess_deadline, orig_pending_txn_size ); + if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; + return start_block_result::succeeded; } - } - try { - size_t orig_pending_txn_size = _pending_incoming_transactions.size(); + } catch ( const guard_exception& e ) { + chain_plugin::handle_guard_exception(e); + return start_block_result::failed; + } catch ( std::bad_alloc& ) { + chain_plugin::handle_bad_alloc(); + } catch ( boost::interprocess::bad_alloc& ) { + chain_plugin::handle_db_exhaustion(); + } + } - // Processing unapplied transactions... - // - if (_producers.empty() && persisted_by_id.empty()) { - // if this node can never produce and has no persisted transactions, - // there is no need for unapplied transactions they can be dropped - chain.get_unapplied_transactions().clear(); - } else { - // derive appliable transactions from unapplied_transactions and drop droppable transactions - unapplied_transactions_type& unapplied_trxs = chain.get_unapplied_transactions(); - if( !unapplied_trxs.empty() ) { - auto unapplied_trxs_size = unapplied_trxs.size(); - int num_applied = 0; - int num_failed = 0; - int num_processed = 0; - auto calculate_transaction_category = [&](const transaction_metadata_ptr& trx) { - if (trx->packed_trx->expiration() < pending_block_time) { - return tx_category::EXPIRED; - } else if (persisted_by_id.find(trx->id) != persisted_by_id.end()) { - return tx_category::PERSISTED; - } else { - return tx_category::UNEXPIRED_UNPERSISTED; - } - }; - - auto itr = unapplied_trxs.begin(); - while( itr != unapplied_trxs.end() ) { - auto itr_next = itr; // save off next since itr may be invalidated by loop - ++itr_next; - - if( preprocess_deadline <= fc::time_point::now() ) exhausted = true; - if( exhausted ) break; - const transaction_metadata_ptr trx = itr->second; - auto category = calculate_transaction_category(trx); - if (category == tx_category::EXPIRED || - (category == tx_category::UNEXPIRED_UNPERSISTED && _producers.empty())) - { - if (!_producers.empty()) { - fc_dlog(_trx_trace_log, "[TRX_TRACE] Node with producers configured is dropping an EXPIRED transaction that was PREVIOUSLY ACCEPTED : ${txid}", - ("txid", trx->id)); - } - itr = unapplied_trxs.erase( itr ); // unapplied_trxs map has not been modified, so simply erase and continue - continue; - } else if (category == tx_category::PERSISTED || - (category == tx_category::UNEXPIRED_UNPERSISTED && _pending_block_mode == pending_block_mode::producing)) - { - ++num_processed; - - try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); - bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && preprocess_deadline < deadline)) { - deadline_is_subjective = true; - deadline = preprocess_deadline; - } - - auto trace = chain.push_transaction(trx, deadline); - if (trace->except) { - if (failure_is_subjective(*trace->except, deadline_is_subjective)) { - exhausted = true; - break; - } else { - // this failed our configured maximum transaction time, we don't want to replay it - // chain.plus_transactions can modify unapplied_trxs, so erase by id - unapplied_trxs.erase( trx->signed_id ); - ++num_failed; - } - } else { - ++num_applied; - } - } LOG_AND_DROP(); - } + return start_block_result::failed; +} - itr = itr_next; - } +void producer_plugin_impl::remove_expired_persisted_trxs( const fc::time_point& deadline ) +{ + auto& persisted_by_expiry = _persistent_transactions.get(); + if (!persisted_by_expiry.empty()) { + chain::controller& chain = chain_plug->chain(); + int num_expired_persistent = 0; + int orig_count = _persistent_transactions.size(); - fc_dlog(_log, "Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}", - ("m", num_processed) - ("n", unapplied_trxs_size) - ("applied", num_applied) - ("failed", num_failed)); - } + bool exhausted = false; + const time_point pending_block_time = chain.pending_block_time(); + while(!persisted_by_expiry.empty() && persisted_by_expiry.begin()->expiry <= pending_block_time) { + if (deadline <= fc::time_point::now()) { + exhausted = true; + break; } - + auto const& txid = persisted_by_expiry.begin()->trx_id; if (_pending_block_mode == pending_block_mode::producing) { - auto& blacklist_by_id = _blacklisted_transactions.get(); - auto& blacklist_by_expiry = _blacklisted_transactions.get(); - auto now = fc::time_point::now(); - if(!blacklist_by_expiry.empty()) { - int num_expired = 0; - int orig_count = _blacklisted_transactions.size(); - - while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= now) { - if (preprocess_deadline <= fc::time_point::now()) break; - blacklist_by_expiry.erase(blacklist_by_expiry.begin()); - num_expired++; - } - - fc_dlog(_log, "Processed ${n} blacklisted transactions, Expired ${expired}", - ("n", orig_count) - ("expired", num_expired)); - } + fc_dlog(_trx_trace_log, "[TRX_TRACE] Block ${block_num} for producer ${prod} is EXPIRING PERSISTED tx: ${txid}", + ("block_num", chain.head_block_num() + 1) + ("prod", chain.pending_block_producer()) + ("txid", txid)); + } else { + fc_dlog(_trx_trace_log, "[TRX_TRACE] Speculative execution is EXPIRING PERSISTED tx: ${txid}", + ("txid", txid)); + } - // scheduled transactions - int num_applied = 0; - int num_failed = 0; - int num_processed = 0; - - auto scheduled_trx_deadline = preprocess_deadline; - if (_max_scheduled_transaction_time_per_block_ms >= 0) { - scheduled_trx_deadline = std::min( - scheduled_trx_deadline, - fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) - ); - } - time_point pending_block_time = chain.pending_block_time(); - const auto& sch_idx = chain.db().get_index(); - const auto scheduled_trxs_size = sch_idx.size(); - auto sch_itr = sch_idx.begin(); - while( sch_itr != sch_idx.end() ) { - if( sch_itr->delay_until > pending_block_time) break; // not scheduled yet - if( sch_itr->published >= pending_block_time ) { - ++sch_itr; - continue; // do not allow schedule and execute in same block - } - if( scheduled_trx_deadline <= fc::time_point::now() ) { - exhausted = true; - break; - } + persisted_by_expiry.erase(persisted_by_expiry.begin()); + num_expired_persistent++; + } - const transaction_id_type trx_id = sch_itr->trx_id; // make copy since reference could be invalidated - if (blacklist_by_id.find(trx_id) != blacklist_by_id.end()) { - ++sch_itr; - continue; - } + if( exhausted ) { + fc_wlog( _log, "Unable to process all ${n} persisted transactions before deadline, Expired ${expired}", + ( "n", orig_count ) + ( "expired", num_expired_persistent ) ); + } else { + fc_dlog( _log, "Processed ${n} persisted transactions, Expired ${expired}", + ( "n", orig_count ) + ( "expired", num_expired_persistent ) ); + } + } +} - auto sch_itr_next = sch_itr; // save off next since sch_itr may be invalidated by loop - ++sch_itr_next; - const auto next_delay_until = sch_itr_next != sch_idx.end() ? sch_itr_next->delay_until : sch_itr->delay_until; - const auto next_id = sch_itr_next != sch_idx.end() ? sch_itr_next->id : sch_itr->id; +void producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point& deadline ) +{ + auto& blacklist_by_expiry = _blacklisted_transactions.get(); + auto now = fc::time_point::now(); + if(!blacklist_by_expiry.empty()) { + int num_expired = 0; + int orig_count = _blacklisted_transactions.size(); + + while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= now) { + if (deadline <= fc::time_point::now()) break; + blacklist_by_expiry.erase(blacklist_by_expiry.begin()); + num_expired++; + } - num_processed++; + fc_dlog( _log, "Processed ${n} blacklisted transactions, Expired ${expired}", + ("n", orig_count)("expired", num_expired) ); + } +} - // configurable ratio of incoming txns vs deferred txns - while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { - if (scheduled_trx_deadline <= fc::time_point::now()) break; +void producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadline ) +{ + chain::controller& chain = chain_plug->chain(); + auto& persisted_by_id = _persistent_transactions.get(); - auto e = _pending_incoming_transactions.front(); - _pending_incoming_transactions.pop_front(); - --orig_pending_txn_size; - _incoming_trx_weight -= 1.0; - process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); - } + // Processing unapplied transactions... + // + if (_producers.empty() && persisted_by_id.empty()) { + // if this node can never produce and has no persisted transactions, + // there is no need for unapplied transactions they can be dropped + chain.get_unapplied_transactions().clear(); + } else { + // derive appliable transactions from unapplied_transactions and drop droppable transactions + unapplied_transactions_type& unapplied_trxs = chain.get_unapplied_transactions(); + if( !unapplied_trxs.empty() ) { + const time_point pending_block_time = chain.pending_block_time(); + auto unapplied_trxs_size = unapplied_trxs.size(); + int num_applied = 0; + int num_failed = 0; + int num_processed = 0; + auto calculate_transaction_category = [&](const transaction_metadata_ptr& trx) { + if (trx->packed_trx->expiration() < pending_block_time) { + return tx_category::EXPIRED; + } else if (persisted_by_id.find(trx->id) != persisted_by_id.end()) { + return tx_category::PERSISTED; + } else { + return tx_category::UNEXPIRED_UNPERSISTED; + } + }; - if (scheduled_trx_deadline <= fc::time_point::now()) { - exhausted = true; - break; + auto itr = unapplied_trxs.begin(); + while( itr != unapplied_trxs.end() ) { + auto itr_next = itr; // save off next since itr may be invalidated by loop + ++itr_next; + + if( deadline <= fc::time_point::now() ) break; + const transaction_metadata_ptr trx = itr->second; + auto category = calculate_transaction_category(trx); + if (category == tx_category::EXPIRED || + (category == tx_category::UNEXPIRED_UNPERSISTED && _producers.empty())) + { + if (!_producers.empty()) { + fc_dlog(_trx_trace_log, "[TRX_TRACE] Node with producers configured is dropping an EXPIRED transaction that was PREVIOUSLY ACCEPTED : ${txid}", + ("txid", trx->id)); } + itr = unapplied_trxs.erase( itr ); // unapplied_trxs map has not been modified, so simply erase and continue + continue; + } else if (category == tx_category::PERSISTED || + (category == tx_category::UNEXPIRED_UNPERSISTED && _pending_block_mode == pending_block_mode::producing)) + { + ++num_processed; try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + auto trx_deadline = fc::time_point::now() + fc::milliseconds( _max_transaction_time_ms ); bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && deadline < trx_deadline)) { deadline_is_subjective = true; - deadline = scheduled_trx_deadline; + trx_deadline = deadline; } - auto trace = chain.push_scheduled_transaction(trx_id, deadline); + auto trace = chain.push_transaction(trx, trx_deadline); if (trace->except) { if (failure_is_subjective(*trace->except, deadline_is_subjective)) { - exhausted = true; break; } else { - auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); - // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist - _blacklisted_transactions.insert(transaction_id_with_expiry{trx_id, expiration}); - num_failed++; + // this failed our configured maximum transaction time, we don't want to replay it + // chain.plus_transactions can modify unapplied_trxs, so erase by id + unapplied_trxs.erase( trx->signed_id ); + ++num_failed; } } else { - num_applied++; + ++num_applied; } } LOG_AND_DROP(); + } - _incoming_trx_weight += _incoming_defer_ratio; - if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; + itr = itr_next; + } - if( sch_itr_next == sch_idx.end() ) break; - sch_itr = sch_idx.lower_bound( boost::make_tuple( next_delay_until, next_id ) ); - } + fc_dlog( _log, "Processed ${m} of ${n} previously applied transactions, Applied ${applied}, Failed/Dropped ${failed}", + ("m", num_processed)("n", unapplied_trxs_size)("applied", num_applied)("failed", num_failed) ); + } + } +} - if( scheduled_trxs_size > 0 ) { - fc_dlog( _log, - "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", - ( "m", num_processed ) - ( "n", scheduled_trxs_size ) - ( "applied", num_applied ) - ( "failed", num_failed ) ); - } +void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_point& deadline, size_t& orig_pending_txn_size ) +{ + chain::controller& chain = chain_plug->chain(); + const time_point pending_block_time = chain.pending_block_time(); + auto& blacklist_by_id = _blacklisted_transactions.get(); + + // scheduled transactions + int num_applied = 0; + int num_failed = 0; + int num_processed = 0; + + auto scheduled_trx_deadline = deadline; + if (_max_scheduled_transaction_time_per_block_ms >= 0) { + scheduled_trx_deadline = std::min( + scheduled_trx_deadline, + fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) + ); + } + const auto& sch_idx = chain.db().get_index(); + const auto scheduled_trxs_size = sch_idx.size(); + auto sch_itr = sch_idx.begin(); + while( sch_itr != sch_idx.end() ) { + if( sch_itr->delay_until > pending_block_time) break; // not scheduled yet + if( sch_itr->published >= pending_block_time ) { + ++sch_itr; + continue; // do not allow schedule and execute in same block + } + if( scheduled_trx_deadline <= fc::time_point::now() ) { + break; + } - } + const transaction_id_type trx_id = sch_itr->trx_id; // make copy since reference could be invalidated + if (blacklist_by_id.find(trx_id) != blacklist_by_id.end()) { + ++sch_itr; + continue; + } - if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit() - return start_block_result::failed; - if (exhausted || preprocess_deadline <= fc::time_point::now()) { - return start_block_result::exhausted; - } else { - // attempt to apply any pending incoming transactions - _incoming_trx_weight = 0.0; + auto sch_itr_next = sch_itr; // save off next since sch_itr may be invalidated by loop + ++sch_itr_next; + const auto next_delay_until = sch_itr_next != sch_idx.end() ? sch_itr_next->delay_until : sch_itr->delay_until; + const auto next_id = sch_itr_next != sch_idx.end() ? sch_itr_next->id : sch_itr->id; - if (!_pending_incoming_transactions.empty()) { - fc_dlog(_log, "Processing ${n} pending transactions", ("n", _pending_incoming_transactions.size())); - while (orig_pending_txn_size && _pending_incoming_transactions.size()) { - if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; - auto e = _pending_incoming_transactions.front(); - _pending_incoming_transactions.pop_front(); - --orig_pending_txn_size; - process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); - } + num_processed++; + + // configurable ratio of incoming txns vs deferred txns + while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { + if (scheduled_trx_deadline <= fc::time_point::now()) break; + + auto e = _pending_incoming_transactions.front(); + _pending_incoming_transactions.pop_front(); + --orig_pending_txn_size; + _incoming_trx_weight -= 1.0; + process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); + } + + if (scheduled_trx_deadline <= fc::time_point::now()) { + break; + } + + try { + auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + bool deadline_is_subjective = false; + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { + deadline_is_subjective = true; + deadline = scheduled_trx_deadline; + } + + auto trace = chain.push_scheduled_transaction(trx_id, deadline); + if (trace->except) { + if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + break; + } else { + auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); + // this failed our configured maximum transaction time, we don't want to replay it add it to a blacklist + _blacklisted_transactions.insert(transaction_id_with_expiry{trx_id, expiration}); + num_failed++; } - return start_block_result::succeeded; + } else { + num_applied++; } + } LOG_AND_DROP(); - } catch ( const guard_exception& e ) { - chain_plugin::handle_guard_exception(e); - return start_block_result::failed; - } catch ( std::bad_alloc& ) { - chain_plugin::handle_bad_alloc(); - } catch ( boost::interprocess::bad_alloc& ) { - chain_plugin::handle_db_exhaustion(); - } + _incoming_trx_weight += _incoming_defer_ratio; + if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; + if( sch_itr_next == sch_idx.end() ) break; + sch_itr = sch_idx.lower_bound( boost::make_tuple( next_delay_until, next_id ) ); } - return start_block_result::failed; + if( scheduled_trxs_size > 0 ) { + fc_dlog( _log, "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", + ( "m", num_processed )( "n", scheduled_trxs_size )( "applied", num_applied )( "failed", num_failed ) ); + } +} + +void producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline, size_t& orig_pending_txn_size ) +{ + if (!_pending_incoming_transactions.empty()) { + fc_dlog(_log, "Processing ${n} pending transactions", ("n", _pending_incoming_transactions.size())); + while (orig_pending_txn_size && _pending_incoming_transactions.size()) { + if( deadline <= fc::time_point::now() ) break; + auto e = _pending_incoming_transactions.front(); + _pending_incoming_transactions.pop_front(); + --orig_pending_txn_size; + process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); + } + } } void producer_plugin_impl::schedule_production_loop() { From d62dd2123d8adcf5a766a837122b2c87fe4a057d Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 16 Sep 2019 18:15:48 -0400 Subject: [PATCH 57/73] Fix accidental code indent change --- plugins/producer_plugin/producer_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index a8b4f87a6c8..41bad8672b0 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -208,7 +208,7 @@ class producer_plugin_impl : public std::enable_shared_from_this Date: Tue, 17 Sep 2019 10:49:24 -0400 Subject: [PATCH 58/73] Added new script + travis execution --- .cicd/submodule-regression-check.sh | 17 ++++++++++++++++- .travis.yml | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh index d0dc7bf117d..a8135440e74 100755 --- a/.cicd/submodule-regression-check.sh +++ b/.cicd/submodule-regression-check.sh @@ -6,20 +6,25 @@ declare -A BASE_MAP if ${TRAVIS:-false}; then BASE_BRANCH=$TRAVIS_BRANCH CURRENT_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} # We default to TRAVIS_BRANCH if it's not a PR so it passes on non PR runs + [[ ! -z $TRAVIS_PULL_REQUEST_SLUG ]] && CURRENT_BRANCH=$TRAVIS_COMMIT # Support git log & echo output else BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-$BUILDKITE_BRANCH} CURRENT_BRANCH=$BUILDKITE_BRANCH fi +[[ $BASE_BRANCH == $CURRENT_BRANCH ]] && echo 'BASE_BRANCH and CURRENT_BRANCH are the same' && exit 0 + echo "getting submodule info for $CURRENT_BRANCH" while read -r a b; do PR_MAP[$a]=$b done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') + echo "getting submodule info for $BASE_BRANCH" git checkout $BASE_BRANCH &> /dev/null git submodule update --init &> /dev/null while read -r a b; do BASE_MAP[$a]=$b done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') + for k in "${!BASE_MAP[@]}"; do base_ts=${BASE_MAP[$k]} pr_ts=${PR_MAP[$k]} @@ -28,7 +33,17 @@ for k in "${!BASE_MAP[@]}"; do echo " timestamp on $BASE_BRANCH: $base_ts" if (( $pr_ts < $base_ts)); then echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating..." - if for c in `git log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"`; do git show --pretty="" --name-only $c; done | grep -q "^$k$"; then + if [[ $TRAVIS == true && ! -z $TRAVIS_PULL_REQUEST_SLUG ]]; then # IF it's a forked PR, we need to switch back to the PR ref/head so we can git log properly + echo "git fetch origin +refs/pull/$TRAVIS_PULL_REQUEST/merge:" + git fetch origin +refs/pull/$TRAVIS_PULL_REQUEST/merge: &> /dev/null + echo "switching back to $TRAVIS_PULL_REQUEST_SLUG:$TRAVIS_PULL_REQUEST_BRANCH ($TRAVIS_COMMIT)" + echo 'git checkout -qf FETCH_HEAD' + git checkout -qf FETCH_HEAD &> /dev/null + elif [[ $BUILDKITE == true ]]; then + echo "switching back to $CURRENT_BRANCH" + git checkout -f $CURRENT_BRANCH &> /dev/null + fi + if [[ ! -z $(for c in $(git --no-pager log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then echo "ERROR: $k has regressed" exit 1 else diff --git a/.travis.yml b/.travis.yml index 110f2756374..bd9c3ee801a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ matrix: - ccache env: - PATH="/usr/local/opt/ccache/libexec:$PATH" -script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh" +script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh && [[ $(uname) != 'Darwin' ]] && ./.cicd/submodule-regression-check.sh" notifications: webhooks: secure: gmqODqoFAil2cR7v++ibqRNECBOSD/VJX+2qPa7XptkVWmVMzbII5CNgBQAscjFsp9arHPMXCCzkBi847PCSiHdsnYFQ4T273FLRWr3cDbLjfmR+BJ7dGKvQnlpSi2Ze2TtAPJyRl+iv+cxDj7cWE5zw2c4xbgh1a/cNO+/ayUfFkyMEIfVWRsHkdkra4gOLywou0XRLHr4CX1V60uU7uuqATnIMMi7gQYwiKKtZqjkbf8wcBvZirDhjQ6lDPN5tnZo6L4QHmqjtzNJg/UrD4h+zES53dLVI4uxlXRAwwpw+mJOFA3QE/3FT+bMQjLCffUz4gZaWcdgebPYzrwSWUbJoFdWAOwcTqivQY0FIQzcz/r6uGWcwWTavzkPEbg68BVM2BZId/0110J6feeTkpJ3MPV+UsIoGTvbg50vi/I06icftuZ/cLqDj3+Emifm7Jlr1sRTSdqtYAJj/2ImUfsb46cwgjAVhFOTvc+KuPgJQgvOXV7bZkxEr5qDWo8Al2sV8BWb83j1rMlZ4LfERokImDVqxu2kkcunchzvhtYFTesSpmwegVpwceCtOtO0rEUgATnfTEHzk2rm8nuz4UtidsQnluUKqmKD0QCqHXFfn+3ZRJsDqr+iCYdxv1BAeAVc9q1L7bgrKDMGiJgkxuhZ2v3J2SflWLvjZjFDduuc= From c1ba6eb70111bae4895835547a8f1e1d83f63020 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 17 Sep 2019 12:02:00 -0400 Subject: [PATCH 59/73] Fix #7767 - node never identified its peer is done syncying --- plugins/net_plugin/net_plugin.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index b563a36294a..d9595e0e7ee 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1552,7 +1552,14 @@ namespace eosio { if (msg.known_blocks.ids.size() == 0) { fc_elog( logger,"got a catch up with ids size = 0" ); } else { - verify_catchup(c, msg.known_blocks.pending, msg.known_blocks.ids.back()); + const block_id_type& id = msg.known_blocks.ids.back(); + controller& cc = chain_plug->chain(); + if( !cc.fetch_block_by_id( id ) ) { + verify_catchup( c, msg.known_blocks.pending, id ); + } else { + // we already have the block, so update peer with our view of the world + c->send_handshake(); + } } } else { From 130cc1c5f3d350d245ce2bbd024ba963b3dbe63f Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 17 Sep 2019 14:26:57 -0400 Subject: [PATCH 60/73] #7766 - report peer address on close --- plugins/net_plugin/net_plugin.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index d9595e0e7ee..47086502a1c 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -810,6 +810,7 @@ namespace eosio { last_handshake_recv = handshake_message(); last_handshake_sent = handshake_message(); my_impl->sync_master->reset_lib_num(shared_from_this()); + fc_ilog(logger, "closing ${a}, ${p}", ("a",peer_addr)("p",peer_name())); fc_dlog(logger, "canceling wait on ${p}", ("p",peer_name())); cancel_wait(); if( read_delay_timer ) read_delay_timer->cancel(); From 235d048ab573fda025d890ff4377c75898ac5ff8 Mon Sep 17 00:00:00 2001 From: Nathan Pierce Date: Tue, 17 Sep 2019 14:58:39 -0400 Subject: [PATCH 61/73] updated script --- .cicd/submodule-regression-check.sh | 33 ++++++++++++++++------------- .travis.yml | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.cicd/submodule-regression-check.sh b/.cicd/submodule-regression-check.sh index a8135440e74..9befa9d0728 100755 --- a/.cicd/submodule-regression-check.sh +++ b/.cicd/submodule-regression-check.sh @@ -4,14 +4,15 @@ declare -A PR_MAP declare -A BASE_MAP # Support Travis and BK if ${TRAVIS:-false}; then + [[ -z $TRAVIS_PULL_REQUEST_BRANCH ]] && echo "Unable to find TRAVIS_PULL_REQUEST_BRANCH ENV. Skipping submodule regression check." && exit 0 BASE_BRANCH=$TRAVIS_BRANCH - CURRENT_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH} # We default to TRAVIS_BRANCH if it's not a PR so it passes on non PR runs - [[ ! -z $TRAVIS_PULL_REQUEST_SLUG ]] && CURRENT_BRANCH=$TRAVIS_COMMIT # Support git log & echo output + CURRENT_BRANCH=$TRAVIS_PULL_REQUEST_BRANCH + [[ ! -z $TRAVIS_PULL_REQUEST_SLUG ]] && CURRENT_BRANCH=$TRAVIS_COMMIT # When we're not running from a PR, the slug is not set. When we are, we need to use the TRAVIS_COMMIT to be sure we're supporting the Forked PR's merge/code that's in the EOS repo. This is needed for the git log below. else - BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-$BUILDKITE_BRANCH} + [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]] && echo "Unable to find BUILDKITE_PULL_REQUEST_BASE_BRANCH ENV. Skipping submodule regression check." && exit 0 + BASE_BRANCH=$BUILDKITE_PULL_REQUEST_BASE_BRANCH CURRENT_BRANCH=$BUILDKITE_BRANCH fi -[[ $BASE_BRANCH == $CURRENT_BRANCH ]] && echo 'BASE_BRANCH and CURRENT_BRANCH are the same' && exit 0 echo "getting submodule info for $CURRENT_BRANCH" while read -r a b; do @@ -25,6 +26,18 @@ while read -r a b; do BASE_MAP[$a]=$b done < <(git submodule --quiet foreach --recursive 'echo $path `git log -1 --format=%ct`') +# We need to switch back to the PR ref/head so we can git log properly +if [[ $TRAVIS == true && ! -z $TRAVIS_PULL_REQUEST_SLUG ]]; then + echo "git fetch origin +refs/pull/$TRAVIS_PULL_REQUEST/merge:" + git fetch origin +refs/pull/$TRAVIS_PULL_REQUEST/merge: &> /dev/null + echo "switching back to $TRAVIS_COMMIT" + echo 'git checkout -qf FETCH_HEAD' + git checkout -qf FETCH_HEAD &> /dev/null +elif [[ $BUILDKITE == true ]]; then + echo "switching back to $CURRENT_BRANCH" + git checkout -f $CURRENT_BRANCH &> /dev/null +fi + for k in "${!BASE_MAP[@]}"; do base_ts=${BASE_MAP[$k]} pr_ts=${PR_MAP[$k]} @@ -32,17 +45,7 @@ for k in "${!BASE_MAP[@]}"; do echo " timestamp on $CURRENT_BRANCH: $pr_ts" echo " timestamp on $BASE_BRANCH: $base_ts" if (( $pr_ts < $base_ts)); then - echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating..." - if [[ $TRAVIS == true && ! -z $TRAVIS_PULL_REQUEST_SLUG ]]; then # IF it's a forked PR, we need to switch back to the PR ref/head so we can git log properly - echo "git fetch origin +refs/pull/$TRAVIS_PULL_REQUEST/merge:" - git fetch origin +refs/pull/$TRAVIS_PULL_REQUEST/merge: &> /dev/null - echo "switching back to $TRAVIS_PULL_REQUEST_SLUG:$TRAVIS_PULL_REQUEST_BRANCH ($TRAVIS_COMMIT)" - echo 'git checkout -qf FETCH_HEAD' - git checkout -qf FETCH_HEAD &> /dev/null - elif [[ $BUILDKITE == true ]]; then - echo "switching back to $CURRENT_BRANCH" - git checkout -f $CURRENT_BRANCH &> /dev/null - fi + echo "$k is older on $CURRENT_BRANCH than $BASE_BRANCH; investigating the difference between $CURRENT_BRANCH and $BASE_BRANCH to look for $k changing..." if [[ ! -z $(for c in $(git --no-pager log $CURRENT_BRANCH ^$BASE_BRANCH --pretty=format:"%H"); do git show --pretty="" --name-only $c; done | grep "^$k$") ]]; then echo "ERROR: $k has regressed" exit 1 diff --git a/.travis.yml b/.travis.yml index bd9c3ee801a..915f636b9e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ matrix: - ccache env: - PATH="/usr/local/opt/ccache/libexec:$PATH" -script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh && [[ $(uname) != 'Darwin' ]] && ./.cicd/submodule-regression-check.sh" +script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh && if [[ $(uname) != 'Darwin' ]]; then ./.cicd/submodule-regression-check.sh; fi" notifications: webhooks: secure: gmqODqoFAil2cR7v++ibqRNECBOSD/VJX+2qPa7XptkVWmVMzbII5CNgBQAscjFsp9arHPMXCCzkBi847PCSiHdsnYFQ4T273FLRWr3cDbLjfmR+BJ7dGKvQnlpSi2Ze2TtAPJyRl+iv+cxDj7cWE5zw2c4xbgh1a/cNO+/ayUfFkyMEIfVWRsHkdkra4gOLywou0XRLHr4CX1V60uU7uuqATnIMMi7gQYwiKKtZqjkbf8wcBvZirDhjQ6lDPN5tnZo6L4QHmqjtzNJg/UrD4h+zES53dLVI4uxlXRAwwpw+mJOFA3QE/3FT+bMQjLCffUz4gZaWcdgebPYzrwSWUbJoFdWAOwcTqivQY0FIQzcz/r6uGWcwWTavzkPEbg68BVM2BZId/0110J6feeTkpJ3MPV+UsIoGTvbg50vi/I06icftuZ/cLqDj3+Emifm7Jlr1sRTSdqtYAJj/2ImUfsb46cwgjAVhFOTvc+KuPgJQgvOXV7bZkxEr5qDWo8Al2sV8BWb83j1rMlZ4LfERokImDVqxu2kkcunchzvhtYFTesSpmwegVpwceCtOtO0rEUgATnfTEHzk2rm8nuz4UtidsQnluUKqmKD0QCqHXFfn+3ZRJsDqr+iCYdxv1BAeAVc9q1L7bgrKDMGiJgkxuhZ2v3J2SflWLvjZjFDduuc= From d41a80bdbf7f8ba8944b6ef4278be6ef4755ff36 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 18 Sep 2019 08:24:56 -0400 Subject: [PATCH 62/73] Fix errorExit string --- tests/nodeos_forked_chain_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nodeos_forked_chain_test.py b/tests/nodeos_forked_chain_test.py index e4e3bc9469b..d8b9553b7d8 100755 --- a/tests/nodeos_forked_chain_test.py +++ b/tests/nodeos_forked_chain_test.py @@ -272,7 +272,7 @@ def getMinHeadAndLib(prodNodes): blockProducer=node.getBlockProducerByNum(blockNum) if producerToSlot[lastBlockProducer]["count"]!=inRowCountPerProducer: - Utils.errorExit("Producer %s, in slot %d, expected to produce %d blocks but produced %d blocks" % (blockProducer, inRowCountPerProducer, producerToSlot[lastBlockProducer]["count"])) + Utils.errorExit("Producer %s, in slot %d, expected to produce %d blocks but produced %d blocks" % (blockProducer, slot, inRowCountPerProducer, producerToSlot[lastBlockProducer]["count"])) if blockProducer==productionCycle[0]: break From 58fee31769d7e4a6b1212fb1eb1367bbdc0c1bbb Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 18 Sep 2019 16:00:55 -0400 Subject: [PATCH 63/73] get_block_id_for_num will throw unknown_block_exception when block num is not found --- plugins/net_plugin/net_plugin.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 47086502a1c..48cead04431 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1502,7 +1502,11 @@ namespace eosio { c->enqueue( note ); } c->syncing = true; - if( cc.get_block_id_for_num( msg.head_num ) != msg.head_id ) { + bool on_fork = true; + try { + on_fork = cc.get_block_id_for_num( msg.head_num ) != msg.head_id; + } catch( ... ) {} + if( on_fork ) { request_message req; req.req_blocks.mode = catch_up; req.req_trx.mode = none; From 87af67e098eb1d1314e331b2f6a8754e6aefdee7 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 19 Sep 2019 09:24:42 -0400 Subject: [PATCH 64/73] Additional cleanup/refactoring --- plugins/producer_plugin/producer_plugin.cpp | 123 ++++++++++++-------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 41bad8672b0..cf4bd077939 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -202,11 +202,11 @@ class producer_plugin_impl : public std::enable_shared_from_this= 0) { + scheduled_trx_deadline = std::min( + scheduled_trx_deadline, + fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) + ); + } + // may exhaust scheduled_trx_deadline but not preprocess_deadline, exhausted preprocess_deadline checked below + process_scheduled_and_incoming_trxs( scheduled_trx_deadline, pending_incoming_process_limit ); } if( app().is_quiting() ) // db guard exception above in LOG_AND_DROP could have called app().quit() @@ -1425,11 +1434,8 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { if( preprocess_deadline <= fc::time_point::now() ) { return start_block_result::exhausted; } else { - _incoming_trx_weight = 0.0; - - // attempt to apply any pending incoming transactions - process_incoming_trxs( preprocess_deadline, orig_pending_txn_size ); - if (preprocess_deadline <= fc::time_point::now()) return start_block_result::exhausted; + if( !process_incoming_trxs( preprocess_deadline, pending_incoming_process_limit ) ) + return start_block_result::exhausted; return start_block_result::succeeded; } @@ -1446,15 +1452,15 @@ producer_plugin_impl::start_block_result producer_plugin_impl::start_block() { return start_block_result::failed; } -void producer_plugin_impl::remove_expired_persisted_trxs( const fc::time_point& deadline ) +bool producer_plugin_impl::remove_expired_persisted_trxs( const fc::time_point& deadline ) { + bool exhausted = false; auto& persisted_by_expiry = _persistent_transactions.get(); if (!persisted_by_expiry.empty()) { chain::controller& chain = chain_plug->chain(); int num_expired_persistent = 0; int orig_count = _persistent_transactions.size(); - bool exhausted = false; const time_point pending_block_time = chain.pending_block_time(); while(!persisted_by_expiry.empty() && persisted_by_expiry.begin()->expiry <= pending_block_time) { if (deadline <= fc::time_point::now()) { @@ -1486,10 +1492,12 @@ void producer_plugin_impl::remove_expired_persisted_trxs( const fc::time_point& ( "expired", num_expired_persistent ) ); } } + return !exhausted; } -void producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point& deadline ) +bool producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point& deadline ) { + bool exhausted = false; auto& blacklist_by_expiry = _blacklisted_transactions.get(); auto now = fc::time_point::now(); if(!blacklist_by_expiry.empty()) { @@ -1497,7 +1505,10 @@ void producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point int orig_count = _blacklisted_transactions.size(); while (!blacklist_by_expiry.empty() && blacklist_by_expiry.begin()->expiry <= now) { - if (deadline <= fc::time_point::now()) break; + if (deadline <= fc::time_point::now()) { + exhausted = true; + break; + } blacklist_by_expiry.erase(blacklist_by_expiry.begin()); num_expired++; } @@ -1505,13 +1516,15 @@ void producer_plugin_impl::remove_expired_blacklisted_trxs( const fc::time_point fc_dlog( _log, "Processed ${n} blacklisted transactions, Expired ${expired}", ("n", orig_count)("expired", num_expired) ); } + return !exhausted; } -void producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadline ) +bool producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadline ) { chain::controller& chain = chain_plug->chain(); auto& persisted_by_id = _persistent_transactions.get(); + bool exhausted = false; // Processing unapplied transactions... // if (_producers.empty() && persisted_by_id.empty()) { @@ -1542,7 +1555,10 @@ void producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin auto itr_next = itr; // save off next since itr may be invalidated by loop ++itr_next; - if( deadline <= fc::time_point::now() ) break; + if( deadline <= fc::time_point::now() ) { + exhausted = true; + break; + } const transaction_metadata_ptr trx = itr->second; auto category = calculate_transaction_category(trx); if (category == tx_category::EXPIRED || @@ -1570,6 +1586,7 @@ void producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin auto trace = chain.push_transaction(trx, trx_deadline); if (trace->except) { if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + exhausted = true; break; } else { // this failed our configured maximum transaction time, we don't want to replay it @@ -1590,9 +1607,10 @@ void producer_plugin_impl::process_unapplied_trxs( const fc::time_point& deadlin ("m", num_processed)("n", unapplied_trxs_size)("applied", num_applied)("failed", num_failed) ); } } + return !exhausted; } -void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_point& deadline, size_t& orig_pending_txn_size ) +bool producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_point& deadline, size_t& pending_incoming_process_limit ) { chain::controller& chain = chain_plug->chain(); const time_point pending_block_time = chain.pending_block_time(); @@ -1602,14 +1620,9 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p int num_applied = 0; int num_failed = 0; int num_processed = 0; + bool exhausted = false; + double incoming_trx_weight = 0.0; - auto scheduled_trx_deadline = deadline; - if (_max_scheduled_transaction_time_per_block_ms >= 0) { - scheduled_trx_deadline = std::min( - scheduled_trx_deadline, - fc::time_point::now() + fc::milliseconds(_max_scheduled_transaction_time_per_block_ms) - ); - } const auto& sch_idx = chain.db().get_index(); const auto scheduled_trxs_size = sch_idx.size(); auto sch_itr = sch_idx.begin(); @@ -1619,7 +1632,8 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p ++sch_itr; continue; // do not allow schedule and execute in same block } - if( scheduled_trx_deadline <= fc::time_point::now() ) { + if( deadline <= fc::time_point::now() ) { + exhausted = true; break; } @@ -1637,31 +1651,36 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p num_processed++; // configurable ratio of incoming txns vs deferred txns - while (_incoming_trx_weight >= 1.0 && orig_pending_txn_size && _pending_incoming_transactions.size()) { - if (scheduled_trx_deadline <= fc::time_point::now()) break; + while (incoming_trx_weight >= 1.0 && pending_incoming_process_limit && _pending_incoming_transactions.size()) { + if (deadline <= fc::time_point::now()) { + exhausted = true; + break; + } auto e = _pending_incoming_transactions.front(); _pending_incoming_transactions.pop_front(); - --orig_pending_txn_size; - _incoming_trx_weight -= 1.0; + --pending_incoming_process_limit; + incoming_trx_weight -= 1.0; process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); } - if (scheduled_trx_deadline <= fc::time_point::now()) { + if (deadline <= fc::time_point::now()) { + exhausted = true; break; } try { - auto deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); + auto trx_deadline = fc::time_point::now() + fc::milliseconds(_max_transaction_time_ms); bool deadline_is_subjective = false; - if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && scheduled_trx_deadline < deadline)) { + if (_max_transaction_time_ms < 0 || (_pending_block_mode == pending_block_mode::producing && deadline < trx_deadline)) { deadline_is_subjective = true; - deadline = scheduled_trx_deadline; + trx_deadline = deadline; } - auto trace = chain.push_scheduled_transaction(trx_id, deadline); + auto trace = chain.push_scheduled_transaction(trx_id, trx_deadline); if (trace->except) { if (failure_is_subjective(*trace->except, deadline_is_subjective)) { + exhausted = true; break; } else { auto expiration = fc::time_point::now() + fc::seconds(chain.get_global_properties().configuration.deferred_trx_expiration_window); @@ -1674,8 +1693,8 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p } } LOG_AND_DROP(); - _incoming_trx_weight += _incoming_defer_ratio; - if (!orig_pending_txn_size) _incoming_trx_weight = 0.0; + incoming_trx_weight += _incoming_defer_ratio; + if (!pending_incoming_process_limit) incoming_trx_weight = 0.0; if( sch_itr_next == sch_idx.end() ) break; sch_itr = sch_idx.lower_bound( boost::make_tuple( next_delay_until, next_id ) ); @@ -1685,20 +1704,26 @@ void producer_plugin_impl::process_scheduled_and_incoming_trxs( const fc::time_p fc_dlog( _log, "Processed ${m} of ${n} scheduled transactions, Applied ${applied}, Failed/Dropped ${failed}", ( "m", num_processed )( "n", scheduled_trxs_size )( "applied", num_applied )( "failed", num_failed ) ); } + return !exhausted; } -void producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline, size_t& orig_pending_txn_size ) +bool producer_plugin_impl::process_incoming_trxs( const fc::time_point& deadline, size_t& pending_incoming_process_limit ) { + bool exhausted = false; if (!_pending_incoming_transactions.empty()) { fc_dlog(_log, "Processing ${n} pending transactions", ("n", _pending_incoming_transactions.size())); - while (orig_pending_txn_size && _pending_incoming_transactions.size()) { - if( deadline <= fc::time_point::now() ) break; + while (pending_incoming_process_limit && _pending_incoming_transactions.size()) { + if( deadline <= fc::time_point::now() ) { + exhausted = true; + break; + } auto e = _pending_incoming_transactions.front(); _pending_incoming_transactions.pop_front(); - --orig_pending_txn_size; + --pending_incoming_process_limit; process_incoming_transaction_async(std::get<0>(e), std::get<1>(e), std::get<2>(e)); } } + return !exhausted; } void producer_plugin_impl::schedule_production_loop() { From c67e7b3c96950e2dc03bfc78aee7c5ef2ba29b5f Mon Sep 17 00:00:00 2001 From: Scott Arnette Date: Fri, 20 Sep 2019 09:37:44 -0400 Subject: [PATCH 65/73] Disabled mongo tests and installation for Mac builds on Travis. --- .cicd/build.sh | 7 +++++-- .travis.yml | 15 ++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index 21cf5ea2953..58337d141ba 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -2,18 +2,21 @@ set -eo pipefail . ./.cicd/helpers/general.sh mkdir -p $BUILD_DIR -CMAKE_EXTRAS="-DBUILD_MONGO_DB_PLUGIN=true -DCMAKE_BUILD_TYPE='Release'" +CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE='Release'" if [[ "$(uname)" == 'Darwin' ]]; then # You can't use chained commands in execute if [[ "$TRAVIS" == 'true' ]]; then export PINNED=false ccache -s CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" - ./$CICD_DIR/platforms/macos-10.14.sh + else + CMAKE_EXTRAS="$CMAKE_EXTRAS -DBUILD_MONGO_DB_PLUGIN=true" fi [[ ! "$PINNED" == 'false' || "$UNPINNED" == 'true' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" cd $BUILD_DIR + echo "cmake $CMAKE_EXTRAS .." cmake $CMAKE_EXTRAS .. + echo "make -j$JOBS" make -j$JOBS else # Linux ARGS=${ARGS:-"--rm --init -v $(pwd):$MOUNTED_DIR"} diff --git a/.travis.yml b/.travis.yml index 915f636b9e7..c482fc1618a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,18 +34,19 @@ matrix: homebrew: update: true packages: - - graphviz + - boost + - python@2 + - python - libtool + - libusb + - graphviz + - automake + - wget - gmp - llvm@4 - pkgconfig - - python - - python@2 - doxygen - - libusb - - openssl - - boost@1.70 - - ccache + - openssl env: - PATH="/usr/local/opt/ccache/libexec:$PATH" script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh && if [[ $(uname) != 'Darwin' ]]; then ./.cicd/submodule-regression-check.sh; fi" From e2d0fd58731b26d45501ad2c2bbd90bd70dec772 Mon Sep 17 00:00:00 2001 From: Scott Arnette Date: Fri, 20 Sep 2019 10:01:22 -0400 Subject: [PATCH 66/73] JQ required for release package test. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c482fc1618a..4ce27f6b473 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,8 @@ matrix: homebrew: update: true packages: + - ccache + - jq - boost - python@2 - python @@ -46,7 +48,7 @@ matrix: - llvm@4 - pkgconfig - doxygen - - openssl + - openssl env: - PATH="/usr/local/opt/ccache/libexec:$PATH" script: "ccache --max-size=1G && ./.cicd/build.sh && ./.cicd/test.sh scripts/parallel-test.sh && ./.cicd/test.sh scripts/serial-test.sh && if [[ $(uname) != 'Darwin' ]]; then ./.cicd/submodule-regression-check.sh; fi" From b92615c0723deea25188a2262b98609b73df4bf6 Mon Sep 17 00:00:00 2001 From: Scott Arnette Date: Fri, 20 Sep 2019 10:03:21 -0400 Subject: [PATCH 67/73] Ensure mongo flag on all Linux builds. --- .cicd/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/build.sh b/.cicd/build.sh index 58337d141ba..8fe4123c50e 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -19,6 +19,7 @@ if [[ "$(uname)" == 'Darwin' ]]; then echo "make -j$JOBS" make -j$JOBS else # Linux + CMAKE_EXTRAS="$CMAKE_EXTRAS -DBUILD_MONGO_DB_PLUGIN=true" ARGS=${ARGS:-"--rm --init -v $(pwd):$MOUNTED_DIR"} . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$IMAGE_TAG.dockerfile PRE_COMMANDS="cd $MOUNTED_DIR/build" From 30398d3f44c4fa17115ad6a98ebbf81e12b5d8a6 Mon Sep 17 00:00:00 2001 From: Scott Arnette Date: Fri, 20 Sep 2019 10:53:37 -0400 Subject: [PATCH 68/73] Current test scripts force mongo to run. Must always install in Travis for now. --- .cicd/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/build.sh b/.cicd/build.sh index 8fe4123c50e..c549a449753 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -9,6 +9,7 @@ if [[ "$(uname)" == 'Darwin' ]]; then export PINNED=false ccache -s CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" + ./$CICD_DIR/platforms/macos-10.14.sh else CMAKE_EXTRAS="$CMAKE_EXTRAS -DBUILD_MONGO_DB_PLUGIN=true" fi From 3f03ede800c92fd113cb0db8650595dbbf9b2db0 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 20 Sep 2019 15:02:22 -0400 Subject: [PATCH 69/73] Remove unneeded request_message --- plugins/net_plugin/net_plugin.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 48cead04431..93ce9f4ae0d 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1483,12 +1483,7 @@ namespace eosio { if (head < msg.head_num ) { fc_dlog(logger, "sync check state 3"); - if (!verify_catchup(c, msg.head_num, msg.head_id)) { - request_message req; - req.req_blocks.mode = catch_up; - req.req_trx.mode = none; - c->enqueue( req ); - } + verify_catchup(c, msg.head_num, msg.head_id); return; } else { From 79c90ff39d4234a32302f5da245f3b4a54140455 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 21 Sep 2019 10:48:02 -0500 Subject: [PATCH 70/73] sync_master recv_block only called on valid blocks, so don't disconnect if unexpected --- plugins/net_plugin/net_plugin.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 93ce9f4ae0d..6f1e4c79bff 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1583,9 +1583,8 @@ namespace eosio { fc_dlog(logger, "got block ${bn} from ${p}",("bn",blk_num)("p",c->peer_name())); if (state == lib_catchup) { if (blk_num != sync_next_expected_num) { - fc_wlog( logger, "expected block ${ne} but got ${bn}, closing connection: ${p}", + fc_wlog( logger, "expected block ${ne} but got ${bn}, from connection: ${p}", ("ne",sync_next_expected_num)("bn",blk_num)("p",c->peer_name()) ); - my_impl->close(c); return; } sync_next_expected_num = blk_num + 1; From 0a613506927c86f143f419732089a512ef46520e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Sat, 21 Sep 2019 20:22:21 -0500 Subject: [PATCH 71/73] Check for out of date last irreversible block notice --- plugins/net_plugin/net_plugin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 6f1e4c79bff..25346501ca0 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1396,8 +1396,8 @@ namespace eosio { sync_known_lib_num = target; } - if (!sync_required()) { - uint32_t bnum = chain_plug->chain().last_irreversible_block_num(); + uint32_t bnum = chain_plug->chain().last_irreversible_block_num(); + if (!sync_required() || target <= bnum) { uint32_t hnum = chain_plug->chain().fork_db_pending_head_block_num(); fc_dlog( logger, "We are already caught up, my irr = ${b}, head = ${h}, target = ${t}", ("b",bnum)("h",hnum)("t",target)); From a79bf671f10766a1a53c7291c4b29116812f99f3 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Tue, 24 Sep 2019 07:41:02 -0500 Subject: [PATCH 72/73] Bump version to 1.8.4 --- CMakeLists.txt | 2 +- README.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a66bdac45d..392d7b4548d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ set( CXX_STANDARD_REQUIRED ON) set(VERSION_MAJOR 1) set(VERSION_MINOR 8) -set(VERSION_PATCH 3) +set(VERSION_PATCH 4) #set(VERSION_SUFFIX develop) if(VERSION_SUFFIX) diff --git a/README.md b/README.md index 9264c12855c..b1e8dae21c4 100644 --- a/README.md +++ b/README.md @@ -44,13 +44,13 @@ $ brew remove eosio #### Ubuntu 18.04 Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.3/eosio_1.8.3-1-ubuntu-18.04_amd64.deb -$ sudo apt install ./eosio_1.8.3-1-ubuntu-18.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.8.4/eosio_1.8.4-1-ubuntu-18.04_amd64.deb +$ sudo apt install ./eosio_1.8.4-1-ubuntu-18.04_amd64.deb ``` #### Ubuntu 16.04 Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.3/eosio_1.8.3-1-ubuntu-16.04_amd64.deb -$ sudo apt install ./eosio_1.8.3-1-ubuntu-16.04_amd64.deb +$ wget https://github.com/eosio/eos/releases/download/v1.8.4/eosio_1.8.4-1-ubuntu-16.04_amd64.deb +$ sudo apt install ./eosio_1.8.4-1-ubuntu-16.04_amd64.deb ``` #### Ubuntu Package Uninstall ```sh @@ -58,8 +58,8 @@ $ sudo apt remove eosio ``` #### Centos RPM Package Install ```sh -$ wget https://github.com/eosio/eos/releases/download/v1.8.3/eosio-1.8.3-1.el7.x86_64.rpm -$ sudo yum install ./eosio-1.8.3-1.el7.x86_64.rpm +$ wget https://github.com/eosio/eos/releases/download/v1.8.4/eosio-1.8.4-1.el7.x86_64.rpm +$ sudo yum install ./eosio-1.8.4-1.el7.x86_64.rpm ``` #### Centos RPM Package Uninstall ```sh From d3ea86b4b8f4f75ace7d8b7a805696b91d14dcbd Mon Sep 17 00:00:00 2001 From: arhag Date: Tue, 24 Sep 2019 10:55:49 -0400 Subject: [PATCH 73/73] Consolidated Security Fixes for 1.8.4 - WebAssembly checktime fixes Co-Authored-By: Matt Witherspoon <32485495+spoonincode@users.noreply.github.com> --- libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp index 2d93bad31d8..65cfa7b552c 100644 --- a/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp +++ b/libraries/chain/include/eosio/chain/wasm_eosio_injection.hpp @@ -754,8 +754,9 @@ namespace eosio { namespace chain { namespace wasm_injections { struct post_op_injectors : wasm_ops::op_types { - using loop_t = wasm_ops::loop ; - using call_t = wasm_ops::call ; + using loop_t = wasm_ops::loop ; + using call_t = wasm_ops::call ; + using grow_memory_t = wasm_ops::grow_memory ; }; template