Add Vagrant based smoke tests (#1067)

* WIP smoke test freebsd

* fix bitrot

We now test that the firewall blocks inbound on host3 from host2

* WIP ipv6 test

* cleanup

* rename to make clear

* fix filename

* restore

* no sudo docker

* WIP

* WIP

* WIP

* WIP

* extra smoke tests

* WIP

* WIP

* add over improvements made in smoke.sh

* more tests

* use generic/freebsd14

* cleanup from test

* smoke test openbsd-amd64

* add netbsd-amd64

* try to fix vagrant

.github/workflows/smoke-extra.yml

@@ -0,0 +1,48 @@
+name: smoke-extra
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, labeled, reopened]
+    paths:
+      - '.github/workflows/smoke**'
+      - '**Makefile'
+      - '**.go'
+      - '**.proto'
+      - 'go.mod'
+      - 'go.sum'
+jobs:
+
+  smoke-extra:
+    if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
+    name: Run extra smoke tests
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+        check-latest: true
+
+    - name: install vagrant
+      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
+
+    - name: freebsd-amd64
+      run: make smoke-vagrant/freebsd-amd64
+
+    - name: openbsd-amd64
+      run: make smoke-vagrant/openbsd-amd64
+
+    - name: netbsd-amd64
+      run: make smoke-vagrant/netbsd-amd64
+
+    - name: linux-386
+      run: make smoke-vagrant/linux-386
+
+    - name: linux-amd64-ipv6disable
+      run: make smoke-vagrant/linux-amd64-ipv6disable
+
+    timeout-minutes: 30

.github/workflows/smoke/build.sh

@@ -11,6 +11,11 @@ mkdir ./build
     cp ../../../../build/linux-amd64/nebula .
     cp ../../../../build/linux-amd64/nebula-cert .
 
+    if [ "$1" ]
+    then
+        cp "../../../../build/$1/nebula" "$1-nebula"
+    fi
+
     HOST="lighthouse1" \
         AM_LIGHTHOUSE=true \
         ../genconfig.sh >lighthouse1.yml

.github/workflows/smoke/genconfig.sh

@@ -47,7 +47,7 @@ listen:
   port: ${LISTEN_PORT:-4242}
 
 tun:
-  dev: ${TUN_DEV:-nebula1}
+  dev: ${TUN_DEV:-tun0}
 
 firewall:
   inbound_action: reject

.github/workflows/smoke/smoke-vagrant.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e -x
+
+set -o pipefail
+
+export VAGRANT_CWD="$PWD/vagrant-$1"
+
+mkdir -p logs
+
+cleanup() {
+    echo
+    echo " *** cleanup"
+    echo
+
+    set +e
+    if [ "$(jobs -r)" ]
+    then
+        docker kill lighthouse1 host2
+    fi
+    vagrant destroy -f
+}
+
+trap cleanup EXIT
+
+CONTAINER="nebula:${NAME:-smoke}"
+
+docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
+docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+
+vagrant up
+vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test"
+
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+sleep 1
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+sleep 1
+vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" &
+sleep 15
+
+# grab tcpdump pcaps for debugging
+docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
+docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap &
+docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
+docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap &
+# vagrant ssh -c "tcpdump -i nebula1 -q -w - -U" 2>logs/host3.inside.log >logs/host3.inside.pcap &
+# vagrant ssh -c "tcpdump -i eth0 -q -w - -U" 2>logs/host3.outside.log >logs/host3.outside.pcap &
+
+docker exec host2 ncat -nklv 0.0.0.0 2000 &
+vagrant ssh -c "ncat -nklv 0.0.0.0 2000" &
+#docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
+#vagrant ssh -c "ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000" &
+
+set +x
+echo
+echo " *** Testing ping from lighthouse1"
+echo
+set -x
+docker exec lighthouse1 ping -c1 192.168.100.2
+docker exec lighthouse1 ping -c1 192.168.100.3
+
+set +x
+echo
+echo " *** Testing ping from host2"
+echo
+set -x
+docker exec host2 ping -c1 192.168.100.1
+# Should fail because not allowed by host3 inbound firewall
+! docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
+
+set +x
+echo
+echo " *** Testing ncat from host2"
+echo
+set -x
+# Should fail because not allowed by host3 inbound firewall
+#! docker exec host2 ncat -nzv -w5 192.168.100.3 2000 || exit 1
+#! docker exec host2 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1
+
+set +x
+echo
+echo " *** Testing ping from host3"
+echo
+set -x
+vagrant ssh -c "ping -c1 192.168.100.1"
+vagrant ssh -c "ping -c1 192.168.100.2"
+
+set +x
+echo
+echo " *** Testing ncat from host3"
+echo
+set -x
+#vagrant ssh -c "ncat -nzv -w5 192.168.100.2 2000"
+#vagrant ssh -c "ncat -nzuv -w5 192.168.100.2 3000" | grep -q host2
+
+vagrant ssh -c "sudo xargs kill </nebula/pid"
+docker exec host2 sh -c 'kill 1'
+docker exec lighthouse1 sh -c 'kill 1'
+sleep 1
+
+if [ "$(jobs -r)" ]
+then
+    echo "nebula still running after SIGTERM sent" >&2
+    exit 1
+fi

.github/workflows/smoke/vagrant-freebsd-amd64/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/freebsd14"
+
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
+end

.github/workflows/smoke/vagrant-linux-386/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial32"
+
+  config.vm.synced_folder "../build", "/nebula"
+end

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -0,0 +1,16 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/jammy64"
+
+  config.vm.synced_folder "../build", "/nebula"
+
+  config.vm.provision :shell do |shell|
+    shell.inline = <<-EOF
+      sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="ipv6.disable=1"/' /etc/default/grub
+      update-grub
+    EOF
+    shell.privileged = true
+    shell.reboot = true
+  end
+end

.github/workflows/smoke/vagrant-netbsd-amd64/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/netbsd9"
+
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
+end

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/openbsd7"
+
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
+end

Makefile

@@ -75,6 +75,8 @@ e2evvvv: e2ev
 e2e-bench: TEST_FLAGS = -bench=. -benchmem -run=^$
 e2e-bench: e2e
 
+DOCKER_BIN = build/linux-amd64/nebula build/linux-amd64/nebula-cert
+
 all: $(ALL:%=build/%/nebula) $(ALL:%=build/%/nebula-cert)
 
 release: $(ALL:%=build/nebula-%.tar.gz)
@@ -212,6 +214,10 @@ smoke-docker-race: BUILD_ARGS = -race
 smoke-docker-race: CGO_ENABLED = 1
 smoke-docker-race: smoke-docker
 
+smoke-vagrant/%: bin-docker build/%/nebula
+	cd .github/workflows/smoke/ && ./build.sh $*
+	cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
+
 .FORCE:
-.PHONY: bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html
+.PHONY: bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
 .DEFAULT_GOAL := bin