db: Fix kill! and stop! semantics (#7)

The semantics of the `kill!` and `stop!` functions was backwards:
`kill!` was sending `SIGTERM`, while `stop!` was sending `SIGKILL`.

Also, modify `stop!` to use `killall -w`, making sure that the process
has terminated by the time the function returns.

.github/workflows/test-build-run.yml

@@ -98,22 +98,12 @@ jobs:
         repository: ${{ inputs.jepsen-cowsql-repo || github.repository }}
         ref: ${{ inputs.jepsen-cowsql-ref || github.ref }}
 
-    - name: Cache Jepsen (lein project) dependencies
-      uses: actions/cache@v3
-      with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-clojure-${{ hashFiles('**/project.clj') }}
-        restore-keys: |
-          ${{ runner.os }}-clojure
-
-    - name: Install Go
-      uses: actions/setup-go@v4
-
     - name: Setup environment
       timeout-minutes: 15
       run: |
         sudo apt update
-        sudo apt install -y gnuplot libsqlite3-dev libuv1-dev liblz4-dev libjna-java graphviz leiningen build-essential
+        sudo apt install golang
+        sudo apt install -y gnuplot libsqlite3-dev libuv1-dev liblz4-dev libjna-java graphviz leiningen build-essential psmisc
         printf core | sudo tee /proc/sys/kernel/core_pattern
 
     - name: Install local Jepsen
@@ -125,17 +115,6 @@ jobs:
         lein install
         cd ../..
 
-    - name: Install libbacktrace
-      run: |
-        git clone https://github.com/ianlancetaylor/libbacktrace --depth 1
-        cd libbacktrace
-        autoreconf -i
-        ./configure
-        make -j4
-        sudo make install
-        sudo ldconfig
-        cd ..
-
     - name: Check out raft
       uses: actions/checkout@v3
       with:
@@ -195,24 +174,14 @@ jobs:
       if: ${{ always() }}
       run: tail -n 100 store/current/jepsen.log > store/current/tail-jepsen.log
 
-    - name: Summary Artifact
-      if: ${{ always() }}
-      uses: actions/upload-artifact@v3
-      with:
-        name: jepsen-data-${{ matrix.workload }}-${{ matrix.nemesis }}-summary
-        path: |
-          store/cowsql*/**/results.edn
-          store/cowsql*/**/latency-raw.png
-          store/cowsql*/**/tail-jepsen.log
-          !**/current/
-          !**/latest/
-
     - name: Failure Artifact
       if: ${{ failure() }}
       uses: actions/upload-artifact@v3
       with:
         name: jepsen-data-${{ matrix.workload }}-${{ matrix.nemesis }}-failure
         path: |
+          /usr/local/lib/libcowsql*
+          /usr/local/lib/libraft*
           store/cowsql*
           !**/current/
           !**/latest/

src/jepsen/cowsql/db.clj

@@ -96,22 +96,26 @@
                       :-cluster (str/join "," (:nodes test)))))
 
 (defn kill!
-  "Gracefully kill, `SIGTERM`, the Go cowsql test application."
+  "Forcefully kill the Go cowsql test application with `SIGKILL`."
   [test node]
-  (let [signal :SIGTERM]
-    (info "Killing" (app-name node) "with" signal "on" node)
-    (cu/grepkill! signal (app-name node))
+  (let [cmd (app-name node)
+        pidfile (app-pidfile test node)]
+    (cu/stop-daemon! cmd pidfile)
     :killed))
 
 (defn stop!
-  "Stop the Go cowsql test application with `stop-daemon!`,
-   which will `SIGKILL`."
+  "Gracefully stop the Go cowsql test application by sending `SIGTERM`."
   [test node]
-  (if (not (cu/daemon-running? (app-pidfile test node)))
-    :not-running
-    (do
-      (cu/stop-daemon! (app-pidfile test node))
-      :stopped)))
+  (let [cmd (app-name node)
+        pidfile (app-pidfile test node)]
+    (info "Send SIGTERM to" cmd)
+    (timeout 30000 (throw+ {:type    ::kill-timed-out
+                            :cmd     cmd
+                            :pidfile pidfile})
+             (meh (c/exec :killall :-TERM :-w cmd)))
+    (when pidfile
+      (meh (c/exec :rm :-rf pidfile)))
+    :stopped))
 
 (defn members
   "Fetch the cluster members from a random node (who will ask the leader)."
@@ -278,7 +282,6 @@
         (kill! test node)
         (when tmpfs
           (db/teardown! tmpfs test node))
-        (Thread/sleep 200) ; avoid race: rm: cannot remove '/opt/cowsql/data': Directory not empty
         (c/exec :rm :-rf (app-dir test node)))
 
       db/LogFiles