Skip to content

Commit

Permalink
Merge pull request #3173 from cloudfoundry/release-mtar
Browse files Browse the repository at this point in the history
feat: release versioned mtar file
  • Loading branch information
bonzofenix authored Oct 3, 2024
2 parents 1d8bf0a + 6f5ad8d commit 2ef916d
Show file tree
Hide file tree
Showing 19 changed files with 419 additions and 204 deletions.
23 changes: 15 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ clean-acceptance:
build: $(all_modules)
build-tests: build-test
build-test: $(addprefix test_,$(go_modules))
build-all: build build-test build-test-app ## Build all modules and tests
build-all: build build-test build-test-app mta-build ## Build all modules and tests
db: target/db
target/db:
@echo "# building $@"
Expand Down Expand Up @@ -260,13 +260,6 @@ build/autoscaler-test.tgz:
@mkdir -p build
@bosh create-release --force --timestamp-version --tarball=build/autoscaler-test.tgz

.PHONY: acceptance-release
acceptance-release: clean-acceptance go-mod-tidy go-mod-vendor build-test-app
@echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' "
@mkdir -p ${DEST}
${AUTOSCALER_DIR}/scripts/compile-acceptance-tests.sh
@tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance'

.PHONY: generate-fakes autoscaler.generate-fakes test-app.generate-fakes
generate-fakes: autoscaler.generate-fakes test-app.generate-fakes
autoscaler.generate-fakes:
Expand Down Expand Up @@ -360,7 +353,21 @@ deploy-prometheus: ${prometheus-bosh-release-path}/manifests
${CI_DIR}/infrastructure/scripts/deploy-prometheus.sh;


.PHONY: mta-release
mta-release: mta-build
@echo " - building mtar release '${VERSION}' to dir: '${DEST}' "

.PHONY: acceptance-release
acceptance-release: clean-acceptance go-mod-tidy go-mod-vendor build-test-app
@echo " - building acceptance test release '${VERSION}' to dir: '${DEST}' "
@mkdir -p ${DEST}
${AUTOSCALER_DIR}/scripts/compile-acceptance-tests.sh
@tar --create --auto-compress --directory="src" --file="${ACCEPTANCE_TESTS_FILE}" 'acceptance'

.PHONY: mta-build
mta-build:
@echo " - building mta"
@make --directory='./src/autoscaler' mta-build

.PHONY: build-test-app
build-test-app:
Expand Down
11 changes: 11 additions & 0 deletions ci/autoscaler/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,14 @@ resources:
regexp: releases/app-autoscaler-acceptance-tests-v(.*).tgz
initial_path: releases/app-autoscaler-acceptance-tests-v0.0.1.tgz

- name: mtar-bucket
type: gcs-resource
source:
bucket: app-autoscaler-releases
json_key: ((autoscaler_releases_uploader_key))
regexp: releases/app-autoscaler-v(.*).mtar
initial_path: releases/app-autoscaler-v0.0.1.mtar

- name: golang-release
type: git
icon: github
Expand Down Expand Up @@ -518,6 +526,9 @@ jobs:
- put: acceptance-tests-bucket
params:
file: "build/artifacts/app-autoscaler-acceptance-tests-v*.tgz"
- put: mtar-bucket
params:
file: "build/artifacts/app-autoscaler-release-v*.mtar"
- put: gh-release
params:
name: build/name
Expand Down
35 changes: 21 additions & 14 deletions ci/autoscaler/scripts/release-autoscaler.sh
Original file line number Diff line number Diff line change
@@ -1,21 +1,8 @@
#! /usr/bin/env bash

# NOTE: you can run this locally for testing !!!
# beware that it adds a commit you need to drop each time also you need to remove dev_releases from root.
#
# DEPLOYMENT=foo \
# GITHUB_TOKEN="ghp_..." \
# PREV_VERSION=12.2.1 \
# DEST="${PWD}/../../../build" \
# VERSION="12.3.0" \
# BUILD_OPTS="--force" \
# AUTOSCALER_CI_BOT_NAME="foo" \
# AUTOSCALER_CI_BOT_EMAIL="[email protected]" \
# AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC="ssh-ed25519 AAAA... [email protected]" \
# AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE="-----BEGIN OPENSSH PRIVATE KEY-----
# b3Bl...
# -----END OPENSSH PRIVATE KEY-----" \
# ./ci/autoscaler/scripts/release-autoscaler.sh
# ./script/local_release_autoscaler.sh


[ -n "${DEBUG}" ] && set -x
Expand Down Expand Up @@ -56,6 +43,17 @@ function create_release() {
--tarball="${build_path}/artifacts/${release_file}"
}

function create_mtar() {
set -e
mkdir -p "${build_path}/artifacts"
local version=$1
local build_path=$2
echo " - creating autorscaler mtar artifact"
pushd "${autoscaler_dir}" > /dev/null
make mta-release VERSION="${version}" DEST="${build_path}/artifacts/"
popd > /dev/null
}

function create_tests() {
set -e
mkdir -p "${build_path}/artifacts"
Expand Down Expand Up @@ -136,19 +134,24 @@ pushd "${autoscaler_dir}" > /dev/null
if [ "${PERFORM_BOSH_RELEASE}" == "true" ]; then
RELEASE_TGZ="app-autoscaler-v${VERSION}.tgz"
ACCEPTANCE_TEST_TGZ="app-autoscaler-acceptance-tests-v${VERSION}.tgz"
AUTOSCALER_MTAR="app-autoscaler-release-v${VERSION}.mtar"
create_release "${VERSION}" "${build_path}" "${RELEASE_TGZ}"
create_tests "${VERSION}" "${build_path}"
create_mtar "${VERSION}" "${build_path}"
[ "${CI}" = "true" ] && commit_release

sha256sum "${build_path}/artifacts/"* > "${build_path}/artifacts/files.sum.sha256"
ACCEPTANCE_SHA256=$( grep "${ACCEPTANCE_TEST_TGZ}$" "${SUM_FILE}" | awk '{print $1}' )
RELEASE_SHA256=$( grep "${RELEASE_TGZ}$" "${SUM_FILE}" | awk '{print $1}')
MTAR_SHA256=$( grep "${AUTOSCALER_MTAR}$" "${SUM_FILE}" | awk '{print $1}')
else
ACCEPTANCE_SHA256="dummy-sha"
RELEASE_SHA256="dummy-sha"
MTAR_SHA256="dummy-sha"
fi
export ACCEPTANCE_SHA256
export RELEASE_SHA256
export MTAR_SHA256

cat >> "${build_path}/changelog.md" <<EOF
Expand All @@ -164,6 +167,10 @@ releases:
version: ${VERSION}
url: https://storage.googleapis.com/app-autoscaler-releases/releases/app-autoscaler-acceptance-tests-v${VERSION}.tgz
sha1: sha256:${ACCEPTANCE_SHA256}
- name: app-autoscaler-mtar
version: ${VERSION}
url: https://storage.googleapis.com/app-autoscaler-releases/releases/app-autoscaler-release-v${VERSION}.mtar
sha1: sha256:${MTAR_SHA256}
\`\`\`
EOF
echo "---------- Changelog file ----------"
Expand Down
3 changes: 3 additions & 0 deletions packages/eventgenerator/spec
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ files:
- autoscaler/vendor/google.golang.org/protobuf/types/descriptorpb/*
- autoscaler/cf/* # gosub
- autoscaler/collection/* # gosub
- autoscaler/configutil/* # gosub
- autoscaler/db/* # gosub
- autoscaler/db/sqldb/* # gosub
- autoscaler/envelopeprocessor/* # gosub
Expand Down Expand Up @@ -43,6 +44,7 @@ files:
- autoscaler/vendor/github.com/cenk/backoff/* # gosub
- autoscaler/vendor/github.com/cenkalti/backoff/v4/* # gosub
- autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub
- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub
- autoscaler/vendor/github.com/facebookgo/clock/* # gosub
- autoscaler/vendor/github.com/felixge/httpsnoop/* # gosub
- autoscaler/vendor/github.com/go-logr/logr/* # gosub
Expand Down Expand Up @@ -81,6 +83,7 @@ files:
- autoscaler/vendor/github.com/klauspost/compress/internal/snapref/* # gosub
- autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub
- autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub
- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub
- autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub
- autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub
- autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub
Expand Down
3 changes: 3 additions & 0 deletions packages/golangapiserver/spec
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ files:
- autoscaler/api/publicapiserver/* # gosub
- autoscaler/api/schedulerclient/* # gosub
- autoscaler/cf/* # gosub
- autoscaler/configutil/* # gosub
- autoscaler/cred_helper/* # gosub
- autoscaler/db/* # gosub
- autoscaler/db/sqldb/* # gosub
Expand All @@ -40,6 +41,7 @@ files:
- autoscaler/vendor/filippo.io/edwards25519/field/* # gosub
- autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub
- autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub
- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub
- autoscaler/vendor/github.com/dlclark/regexp2/* # gosub
- autoscaler/vendor/github.com/dlclark/regexp2/syntax/* # gosub
- autoscaler/vendor/github.com/fatih/color/* # gosub
Expand Down Expand Up @@ -85,6 +87,7 @@ files:
- autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub
- autoscaler/vendor/github.com/mattn/go-colorable/* # gosub
- autoscaler/vendor/github.com/mattn/go-isatty/* # gosub
- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub
- autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub
- autoscaler/vendor/github.com/ogen-go/ogen/conv/* # gosub
- autoscaler/vendor/github.com/ogen-go/ogen/http/* # gosub
Expand Down
3 changes: 3 additions & 0 deletions packages/operator/spec
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ files:
- autoscaler/vendor/*
- autoscaler/operator/db/*
- autoscaler/cf/* # gosub
- autoscaler/configutil/* # gosub
- autoscaler/db/* # gosub
- autoscaler/db/sqldb/* # gosub
- autoscaler/healthendpoint/* # gosub
Expand All @@ -29,6 +30,7 @@ files:
- autoscaler/vendor/filippo.io/edwards25519/field/* # gosub
- autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub
- autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub
- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub
- autoscaler/vendor/github.com/go-logr/logr/* # gosub
- autoscaler/vendor/github.com/go-logr/logr/funcr/* # gosub
- autoscaler/vendor/github.com/go-logr/stdr/* # gosub
Expand Down Expand Up @@ -62,6 +64,7 @@ files:
- autoscaler/vendor/github.com/klauspost/compress/internal/snapref/* # gosub
- autoscaler/vendor/github.com/klauspost/compress/zstd/* # gosub
- autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub
- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub
- autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub
- autoscaler/vendor/github.com/openzipkin/zipkin-go/idgenerator/* # gosub
- autoscaler/vendor/github.com/openzipkin/zipkin-go/model/* # gosub
Expand Down
3 changes: 3 additions & 0 deletions packages/scalingengine/spec
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ files:
- autoscaler/vendor/*
- autoscaler/scalingengine/db/*
- autoscaler/cf/* # gosub
- autoscaler/configutil/* # gosub
- autoscaler/db/* # gosub
- autoscaler/db/sqldb/* # gosub
- autoscaler/healthendpoint/* # gosub
Expand All @@ -32,6 +33,7 @@ files:
- autoscaler/vendor/filippo.io/edwards25519/field/* # gosub
- autoscaler/vendor/github.com/beorn7/perks/quantile/* # gosub
- autoscaler/vendor/github.com/cespare/xxhash/v2/* # gosub
- autoscaler/vendor/github.com/cloud-gov/go-cfenv/* # gosub
- autoscaler/vendor/github.com/dlclark/regexp2/* # gosub
- autoscaler/vendor/github.com/dlclark/regexp2/syntax/* # gosub
- autoscaler/vendor/github.com/fatih/color/* # gosub
Expand Down Expand Up @@ -76,6 +78,7 @@ files:
- autoscaler/vendor/github.com/klauspost/compress/zstd/internal/xxhash/* # gosub
- autoscaler/vendor/github.com/mattn/go-colorable/* # gosub
- autoscaler/vendor/github.com/mattn/go-isatty/* # gosub
- autoscaler/vendor/github.com/mitchellh/mapstructure/* # gosub
- autoscaler/vendor/github.com/munnerz/goautoneg/* # gosub
- autoscaler/vendor/github.com/ogen-go/ogen/conv/* # gosub
- autoscaler/vendor/github.com/ogen-go/ogen/http/* # gosub
Expand Down
71 changes: 71 additions & 0 deletions scripts/local_release_autoscaler.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#! /usr/bin/env bash
#
# shellcheck disable=SC1091
#
set -euo pipefail
script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )

DEPLOYMENT=foo
DEBUG=true
DEST="${script_dir}/../build"
BUILD_OPTS="--force"
AUTOSCALER_CI_BOT_NAME="foo"
AUTOSCALER_CI_BOT_EMAIL="[email protected]"
PREV_VERSION="$(yq ".properties.\"autoscaler.apiserver.info.build\".default" jobs/golangapiserver/spec)"

VERSION="$(cat "${script_dir}/../VERSION")-pre"

export DEPLOYMENT
export DEBUG
export DEST
export BUILD_OPTS
export AUTOSCALER_CI_BOT_NAME
export AUTOSCALER_CI_BOT_EMAIL
export PREV_VERSION
export VERSION

# check for GITHUB_TOKEN
if [ -z "${GITHUB_TOKEN}" ]; then
echo "GITHUB_TOKEN is not set"
exit 1
fi

find_or_create_ssh_key() {
if [ -f ~/.ssh/id_ed25519 ]; then
echo "ssh key already exists"
return
fi

ssh-keygen -t ed25519 -C "${AUTOSCALER_CI_BOT_EMAIL}" -f ~/.ssh/id_ed25519 -N ""
}

prerelease() {
pushd "${script_dir}/.." > /dev/null
make clean generate-fakes generate-openapi-generated-clients-and-servers go-mod-tidy go-mod-vendor db scheduler
popd > /dev/null
}

delete_dev_releases() {
rm -rf dev_releases
}


release_autoscaler() {
AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC=$(cat ~/.ssh/id_ed25519.pub)
AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE=$(cat ~/.ssh/id_ed25519)
export AUTOSCALER_CI_BOT_SIGNING_KEY_PUBLIC
export AUTOSCALER_CI_BOT_SIGNING_KEY_PRIVATE

source "${script_dir}/../ci/autoscaler/scripts/release-autoscaler.sh"
echo "beware that it adds a commit you need to drop each time also you need to remove dev_releases from root."
}

main() {
find_or_create_ssh_key
delete_dev_releases
prerelease
release_autoscaler
}

main

4 changes: 1 addition & 3 deletions src/acceptance/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,16 @@ require (
github.com/cloudfoundry/cf-test-helpers/v2 v2.9.0
github.com/onsi/ginkgo/v2 v2.20.2
github.com/onsi/gomega v1.34.2
github.com/stretchr/testify v1.9.0
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/stretchr/testify v1.9.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
Expand Down
14 changes: 0 additions & 14 deletions src/acceptance/helpers/helpers_test.go

This file was deleted.

1 change: 1 addition & 0 deletions src/autoscaler/.gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
build/
fakes/
mta_archives/
mta.yaml
11 changes: 10 additions & 1 deletion src/autoscaler/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ SHELL := /bin/bash
MAKEFLAGS := -s
aes_terminal_font_yellow := \e[38;2;255;255;0m
aes_terminal_reset := \e[0m
VERSION ?= 0.0.0-rc.1
DEST ?= build

GO_VERSION = $(shell go version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/')
GO_DEPENDENCIES = $(shell find . -type f -name '*.go')
Expand All @@ -29,6 +31,7 @@ GINKGO_OPTS = -r --race --require-suite --randomize-all --cover ${OPTS}
GINKGO_VERSION = v$(shell cat ../../.tool-versions | grep ginkgo | cut --delimiter=' ' --fields='2')



# ogen generated OpenAPI clients and servers
openapi-generated-clients-and-servers-dir := ./helpers/apis/scalinghistory
openapi-spec-path := ../../api
Expand Down Expand Up @@ -149,7 +152,8 @@ clean:
mta-deploy: mta-build build-extension-file
$(MAKE) -f metricsforwarder/Makefile set-security-group
@echo "Deploying with extension file: $(EXTENSION_FILE)"
@cf deploy mta_archives/*.mtar -f --delete-services -e $(EXTENSION_FILE)
@cf deploy $(MAKEFILE_DIR)/$(DEST)/*.mtar -f --delete-services -e $(EXTENSION_FILE)


build-extension-file:
echo "extension file at: $(EXTENSION_FILE)"
Expand All @@ -162,7 +166,12 @@ mta-logs:

.PHONY: mta-build
mta-build: mta-build-clean
@echo "bulding mtar file for version: $(VERSION)"
cp mta.tpl.yaml mta.yaml
sed -i 's/VERSION/$(VERSION)/g' mta.yaml
mkdir -p $(DEST)
mbt build
@mv mta_archives/com.github.cloudfoundry.app-autoscaler-release_$(VERSION).mtar $(DEST)/app-autoscaler-release-v$(VERSION).mtar

mta-build-clean:
rm -rf mta_archives
Loading

0 comments on commit 2ef916d

Please sign in to comment.