Adds swap space for ALI runners and report to ci and metrics if it is…

… used (#6058)

Adds a swap space for the autoscaled runners.

Prints on post_job step if the swap usage was detected during the job
running, and sends metrics related to swap usage per job.

terraform-aws-github-runner/modules/runners-instances/templates/install-config-runner.sh

@@ -85,6 +85,9 @@ case \$ami_id in
   *) echo "AMI Name: unknown";;
 esac
 
+grep 'pswpin' /proc/vmstat | awk '{print $2}' >/tmp/pswpin_before_job || true
+grep 'pswpout' /proc/vmstat | awk '{print $2}' >/tmp/pswpout_before_job || true
+
 echo "AMI ID: \$ami_id"
 
 metric_report "runner_scripts.before_job" 1
@@ -94,6 +97,21 @@ EOF
 #!/bin/bash
 . /home/$USER_NAME/runner-scripts/utils.sh
 
+grep 'pswpin' /proc/vmstat | awk '{print $2}' >/tmp/pswpin_after_job || true
+grep 'pswpout' /proc/vmstat | awk '{print $2}' >/tmp/pswpout_after_job || true
+
+if cmp --silent /tmp/pswpin_before_job /tmp/pswpin_after_job ; then
+  echo "[!ALERT!] Swap in detected! [!ALERT!]"
+  metric_report "runner_scripts.swap_in" 1
+  metric_report "runner_scripts.swap_op" 1
+fi
+
+if cmp --silent /tmp/pswpout_before_job /tmp/pswpout_after_job ; then
+  echo "[!ALERT!] Swap out detected [!ALERT!]"
+  metric_report "runner_scripts.swap_out" 1
+  metric_report "runner_scripts.swap_op" 1
+fi
+
 sudo chown -R $USER_NAME:$USER_NAME /home/$USER_NAME/actions-runner
 metric_report "runner_scripts.after_job" 1
 EOF

terraform-aws-github-runner/modules/runners-instances/templates/user-data.sh

@@ -138,6 +138,11 @@ fi
 
 ${post_install}
 
+sudo fallocate -l 3G /swapfile
+sudo chmod 600 /swapfile
+sudo mkswap /swapfile
+sudo swapon /swapfile
+
 ./svc.sh start
 
 metric_report "linux_userdata.success" 1