feat(metricsforwarder): Directly write to Log Cache via Syslog protocol

.github/workflows/acceptance_tests_reusable.yaml

@@ -136,6 +136,7 @@ jobs:
 
   deployment_cleanup:
     needs: [ acceptance_tests_verify ]
+    if: "!contains(github.event.pull_request.labels.*.name, 'skip-cleanup')"
     name: Deployment cleanup
     runs-on: ubuntu-latest
     container:

.gitignore

@@ -60,3 +60,5 @@ keys/
 # easy sharing, see:
 # <https://code.visualstudio.com/docs/editor/extension-marketplace#_workspace-recommended-extensions>
 !.vscode/extensions.json
+
+.devbox

.tool-versions

@@ -1,6 +1,7 @@
 act 0.2.62
 actionlint 1.6.27
 bosh 7.3.1
+credhub 2.9.29
 cf 8.7.10
 concourse 7.10.0
 direnv 2.34.0
@@ -14,3 +15,4 @@ maven 3.8.6
 ruby 3.3.1
 shellcheck 0.10.0
 yq 4.43.1
+gum 0.13.0

Makefile

@@ -79,7 +79,7 @@ clean-scheduler:
 	@rm -rf src/scheduler/src/test/resources/certs
 clean-certs:
 	@echo " - cleaning test certs"
-	@rm -f testcerts/*
+	@rm -f test-certs/*
 clean-bosh-release:
 	@echo " - cleaning bosh dev releases"
 	@rm -rf dev_releases
@@ -231,7 +231,7 @@ lint: $(addprefix lint_,$(go_modules))  rubocop
 rubocop:
 	@echo " - ruby scripts"
 	@bundle install
-	@bundle exec rubocop ./spec ./packages
+	@bundle exec rubocop ${RUBOCOP_OPTS} ./spec ./packages
 
 .PHONY: markdownlint
 markdownlint: markdownlint-cli
@@ -305,7 +305,7 @@ mod-download:
 
 .PHONY: acceptance.go-mod-vendor autoscaler.go-mod-vendor changelog.go-mod-vendor \
 				changeloglockcleander.go-mod-vendor
-go-mod-vendor: acceptance.go-mod-vendor autoscaler.go-mod-vendor changelog.go-mod-vendor \
+go-mod-vendor: clean-vendor acceptance.go-mod-vendor autoscaler.go-mod-vendor changelog.go-mod-vendor \
 							 changeloglockcleander.go-mod-vendor
 acceptance.go-mod-vendor:
 	make --directory='${go-acceptance-dir}' go-mod-vendor

README.md

@@ -302,6 +302,21 @@ database.password | The password of the user specified above in "database.userna
 database.sslmode | There are 6 values allowed for "postgres": disable, allow, prefer, require, verify-ca and verify-full. Please refer to [Postgres SSL definition](https://www.postgresql.org/docs/current/libpq-ssl.html) when define `database_sslmode`.  For "mysql", there are 7 values allowed: false, true, skip-verify, preferred, verify-ca, verify_identity.Please refer to [Mysql SSL definition(Golang)](https://github.com/go-sql-driver/mysql#tls) and [Mysql Connector SSL](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-using-ssl.html)
 database.tls.ca | PEM-encoded certification authority for secure TLS communication. Only required when sslmode is verify-ca or verify-full(postgres) or verify_identity(mysql) and can be omitted for other sslmode.
 
+## Run linting
+
+Linting can be run through make:
+
+```
+make lint
+```
+
+Autofix can be trigger by providing the following options:
+
+
+```
+OPTS=--fix RUBOCOP_OPTS=-A make lint
+```
+
 ## Register service
 
 Log in to Cloud Foundry with admin user, and use the following commands to register `app-autoscaler` service

ci/Makefile

@@ -0,0 +1,11 @@
+
+.PHONY: set-autoscaler-pipeline
+
+set-autoscaler-pipeline:
+	@./autoscaler/set-pipeline.sh
+
+.PHONY: unpause-pipeline
+
+unpause-pipeline:
+	@./scripts/unpause-pipeline.sh
+

ci/README.md

@@ -1,13 +1,13 @@
 # app-autoscaler-ci
 
 This repository provides all public scripts and pipeline deployments used
-by the app autoscaler team.  The public pipeline is hosted at: <https://bosh.ci.cloudfoundry.org/>.
+by the app autoscaler team.  The public pipeline is hosted at: <https://concourse.app-runtime-interfaces.ci.cloudfoundry.org>.
 
 To reproduce this pipeline, you can use your own private configuration files for the `pipeline.yml` files as described below.
 
 ## Autoscaler
 
-This directory contains the concourse `pipeline.yml` for the autoscaler [pipeline](https://bosh.ci.cloudfoundry.org/pipelines/app-autoscaler)
+This directory contains the concourse `pipeline.yml` for the autoscaler [pipeline](https://concourse.app-runtime-interfaces.ci.cloudfoundry.org/teams/app-autoscaler/pipelines/app-autoscaler-release)
 and all of the associated scripts. To use this manifest, you need to provide a private configuration file
 for all of the template parameters.
 
@@ -28,10 +28,15 @@ This directory contains the terragrunt managed stacks of resouces in account app
 __Setup__
 
 ```
-fly --target autoscaler login --team-name app-autoscaler --concourse-url https://bosh.ci.cloudfoundry.org/
-push autoscaler
-./set-pipeline.sh
-popd
+fly --target app-autoscaler-release login --team-name app-autoscaler --concourse-url https://concourse.app-runtime-interfaces.ci.cloudfoundry.org
+make set-autoscaler-pipeline
+```
+
+## Unpause pipeline and jobs
+
+```
+# You will be prompted to select the specific jobs you want to unpause.
+make unpause-pipeline
 ```
 
 ## Prometheus

ci/autoscaler/pipeline.yml

@@ -12,7 +12,7 @@ anchors:
       operations/set-release-version.yml
       operations/enable-scheduler-logging.yml
 
-  app-autoscaler-ops-files-log-cache: &app-autoscaler-ops-files-log-cache
+  app-autoscaler-ops-files-log-cache-metron: &app-autoscaler-ops-files-log-cache-metron
     OPS_FILES: |
       operations/add-releases.yml
       operations/instance-identity-cert-from-cf.yml
@@ -22,6 +22,22 @@ anchors:
       operations/add-extra-plan.yml
       operations/set-release-version.yml
       operations/configure-log-cache-and-forward-metrics-via-mtls.yml
+      operations/enable-metricsforwarder-via-metron-agent.yml
+      operations/remove-metricsserver.yml
+      operations/remove-metricsgateway.yml
+      operations/enable-scheduler-logging.yml
+
+  app-autoscaler-ops-files-log-cache-syslog: &app-autoscaler-ops-files-log-cache-syslog
+    OPS_FILES: |
+      operations/add-releases.yml
+      operations/instance-identity-cert-from-cf.yml
+      operations/add-postgres-variables.yml
+      operations/enable-nats-tls.yml
+      operations/loggregator-certs-from-cf.yml
+      operations/add-extra-plan.yml
+      operations/set-release-version.yml
+      operations/configure-log-cache-and-forward-metrics-via-mtls.yml
+      operations/enable-metricsforwarder-via-syslog-agent.yml
       operations/remove-metricsserver.yml
       operations/remove-metricsgateway.yml
       operations/enable-scheduler-logging.yml
@@ -42,7 +58,8 @@ groups:
 - name: all
   jobs:
   - acceptance
-  - acceptance-log-cache
+  - acceptance-log-cache-metron
+  - acceptance-log-cache-syslog
   - cleanup-autoscaler-deployments
   - fetch-latest-stemcell
   - draft
@@ -57,7 +74,8 @@ groups:
 - name: autoscaler-release
   jobs:
   - acceptance
-  - acceptance-log-cache
+  - acceptance-log-cache-metron
+  - acceptance-log-cache-syslog
   - draft
   - integration-tests
   - release
@@ -177,7 +195,8 @@ jobs:
     vars:
       branch_name: ((branch_name))
       acceptance_deployment_name: ((acceptance_deployment_name))
-      logcache_acceptance_deployment_name: ((logcache_acceptance_deployment_name))
+      acceptance_deployment_name_logcache_metron: ((acceptance_deployment_name_logcache_metron))
+      acceptance_deployment_name_logcache_syslog: ((acceptance_deployment_name_logcache_syslog))
 
 - name: unit-tests
   public: true
@@ -260,15 +279,68 @@ jobs:
         SUITES: broker
       timeout: 15m
 
-- name: acceptance-log-cache
+- name: acceptance-log-cache-metron
+  public: true
+  build_logs_to_retain: 100
+  serial: true
+  on_success:
+    task: cleanup
+    file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml
+    params: &acceptance-log-cache-params
+      DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_metron))
+  plan:
+  - in_parallel:
+    - get: bbl-state
+    - get: app-autoscaler-release
+      passed: [unit-tests, integration-tests]
+      trigger: true
+    - get: ci
+  - task: make-prerelease
+    file: ci/ci/autoscaler/tasks/make/make.yaml
+    params:
+      # ⚠️ Here it is used that make officially guarantees to reach the goals in the provided order.
+      TARGETS: generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler
+    timeout: 15m
+  - task: deploy-autoscaler
+    file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml
+    params:
+      <<: *acceptance-log-cache-params
+      <<: *app-autoscaler-ops-files-log-cache-metron
+    timeout: 30m
+  - task: register-broker
+    file: ci/ci/autoscaler/tasks/register-broker.yml
+    params:
+      <<: *acceptance-log-cache-params
+    timeout: 5m
+  - in_parallel:
+    - task: autoscaler-acceptance-api
+      file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml
+      params:
+        <<: *acceptance-log-cache-params
+        SUITES: api
+      timeout: 15m
+    - task: autoscaler-acceptance-app
+      file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml
+      params:
+        <<: *acceptance-log-cache-params
+        SUITES: app
+      timeout: 45m
+    - task: autoscaler-acceptance-broker
+      file: ci/ci/autoscaler/tasks/run-acceptance-tests.yml
+      params:
+        <<: *acceptance-log-cache-params
+        SUITES: broker
+      timeout: 15m
+
+- name: acceptance-log-cache-syslog
   public: true
   build_logs_to_retain: 100
   serial: true
   on_success:
     task: cleanup
     file: ci/ci/autoscaler/tasks/cleanup-autoscaler.yml
     params: &acceptance-log-cache-params
-      DEPLOYMENT_NAME: ((logcache_acceptance_deployment_name))
+      DEPLOYMENT_NAME: ((acceptance_deployment_name_logcache_syslog))
   plan:
   - in_parallel:
     - get: bbl-state
@@ -286,7 +358,7 @@ jobs:
     file: ci/ci/autoscaler/tasks/deploy-autoscaler.yml
     params:
       <<: *acceptance-log-cache-params
-      <<: *app-autoscaler-ops-files-log-cache
+      <<: *app-autoscaler-ops-files-log-cache-syslog
     timeout: 30m
   - task: register-broker
     file: ci/ci/autoscaler/tasks/register-broker.yml
@@ -374,7 +446,8 @@ jobs:
     - get: app-autoscaler-release
       passed:
       - acceptance
-      - acceptance-log-cache
+      - acceptance-log-cache-metron
+      - acceptance-log-cache-syslog
       trigger: true
     - get: previous-stable-release
   - task: deploy-previous-stable-release

ci/autoscaler/scripts/deploy-autoscaler.sh

@@ -20,6 +20,7 @@ ops_files=${OPS_FILES:-"${autoscaler_dir}/operations/add-releases.yml\
   ${autoscaler_dir}/operations/remove-metricsserver.yml\
   ${autoscaler_dir}/operations/remove-metricsgateway.yml\
   ${autoscaler_dir}/operations/enable-log-cache-via-uaa.yml\
+  ${autoscaler_dir}/operations/enable-metricsforwarder-via-syslog-agent.yml\
   ${autoscaler_dir}/operations/enable-scheduler-logging.yml"}
 
 

ci/autoscaler/set-pipeline.sh

@@ -11,7 +11,7 @@ if "$(which gh)" > /dev/null; then
   echo "gh cli found"
   gh --version
 else
-  echo "no gh cli found!"  
+  echo "no gh cli found!"
   exit 1
 fi
 
@@ -31,10 +31,12 @@ function set_pipeline(){
   add_var branch_name "${CURRENT_BRANCH}"
   if [[ -z $PR_NUMBER ]]; then
     add_var acceptance_deployment_name          "acceptance"
-    add_var logcache_acceptance_deployment_name "acceptance-lc"
+    add_var acceptance_deployment_name_logcache_metron "acceptance-lc"
+    add_var acceptance_deployment_name_logcache_syslog "acceptance-lc-sl"
   else
     add_var acceptance_deployment_name          "${PR_NUMBER}-acceptance"
-    add_var logcache_acceptance_deployment_name "${PR_NUMBER}-acceptance-lc"
+    add_var acceptance_deployment_name_logcache_metron "${PR_NUMBER}-acceptance-lc"
+    add_var acceptance_deployment_name_logcache_syslog "${PR_NUMBER}-acceptance-lc-sl"
   fi
 
   # shellcheck disable=SC2086

ci/scripts/unpause-pipeline.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2086
+#
+
+target="app-autoscaler-release"
+
+function unpause-job(){
+  pipeline="$1"
+  jobs=$(fly -t "$target" jobs -p "$pipeline" --json | jq ".[] | select(.paused==true) | .name" -r)
+
+  if [[ -z "$jobs" ]]; then
+    echo "No paused job in pipeline $pipeline"
+    return
+  fi
+
+  selected_job=$(gum choose --no-limit $jobs --header "Select jobs to unpause from pipeline $pipeline")
+
+
+  if [[ -z "$selected_job" ]]; then
+    echo "No job selected to unpause"
+    return
+  fi
+
+  for j in $selected_job; do
+    fly -t "$target" unpause-job -j "$pipeline/$j"
+  done
+}
+
+function unpause-pipeline(){
+  payload=$(fly -t "$target" pipelines --json)
+
+  pipelines=$(echo "$payload" | jq ".[] |.name" -r | sort)
+  # ignore shellcheck warning
+  pipeline=$(gum choose $pipelines "all")
+
+  if [[ "$pipeline" == "all" ]]; then
+    for p in $pipelines; do
+      fly -t "$target" unpause-pipeline -p "$p"
+      unpause-job "$p"
+    done
+  else
+    fly -t "$target" unpause-pipeline -p "$pipeline"
+    unpause-job "$pipeline"
+  fi
+
+}
+
+function check-login(){
+  if ! fly -t "$target" status;  then
+    echo
+    echo "fly -t $target login"
+    echo
+    exit 1
+  fi
+}
+
+check-login
+unpause-pipeline "${@:-}"

devbox.json

@@ -4,7 +4,6 @@
     "path:local-flake#app-autoscaler-cli-plugin",
     "path:local-flake#log-cache-cli-plugin",
     "path:local-flake#uaac",
-    "credhub-cli@latest",
     "delve@latest",
     "gh@latest",
     "go-tools@latest",
@@ -33,6 +32,8 @@
     "[email protected]",
     "bundix@latest",
     "oha@latest",
+    "[email protected]",
+    "[email protected]",
     "google-cloud-sdk@latest",
     "ginkgo@latest",
     "temurin-bin-17@latest"