From fb5c75ba3f16656b7d2751cfa5a5c386ebd9cb25 Mon Sep 17 00:00:00 2001 From: Gavin Inglis Date: Fri, 28 Jul 2023 15:47:28 -0700 Subject: [PATCH] ci: add script to update rootfs URL for Finch on Windows runfinch/finch#492, we need to provide a rootfs to WSL2. This rootfs lives in our dependencies bucket, and will be updated from time-to-time for security patches, bug fixes, etc. This commit will automatically pull the most recent rootfs from the depenedencies bucket as part of the Update Deps action. This commit also updates the upload rootfs action to use better pathing than storing the rootfs at root level of S3 bucket. Signed-off-by: Gavin Inglis --- .github/workflows/rootfs.yaml | 8 +++++- .github/workflows/update-dependencies.yaml | 1 + Makefile | 6 +++++ bin/update-rootfs.sh | 29 ++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100755 bin/update-rootfs.sh diff --git a/.github/workflows/rootfs.yaml b/.github/workflows/rootfs.yaml index 8cbec71..bc7c63b 100644 --- a/.github/workflows/rootfs.yaml +++ b/.github/workflows/rootfs.yaml @@ -20,6 +20,7 @@ jobs: strategy: matrix: arch: ['amd64', 'arm64'] + platform: ['common'] steps: - name: Generate Timestamp id: timestamp @@ -63,4 +64,9 @@ jobs: zstd -z -18 finch-rootfs-production-${{ matrix.arch }}.tar -o finch-rootfs-production-${{ matrix.arch }}-"$TIMESTAMP".tar.zst - aws s3 cp ./finch-rootfs-production-${{ matrix.arch }}-"$TIMESTAMP".tar.zst s3://${{ secrets.DEPENDENCY_BUCKET_NAME }} + ARCHPATH="x86-64" + if [ ${{ matrix.arch }} == "arm64" ]; then + ARCHPATH="aarch64" + fi + + aws s3 cp ./finch-rootfs-production-${{ matrix.arch }}-"$TIMESTAMP".tar.zst s3://${{ secrets.DEPENDENCY_BUCKET_NAME }}/${{ matrix.platform }}/$ARCHPATH/ diff --git a/.github/workflows/update-dependencies.yaml b/.github/workflows/update-dependencies.yaml index d0101c1..239a26f 100644 --- a/.github/workflows/update-dependencies.yaml +++ b/.github/workflows/update-dependencies.yaml @@ -29,6 +29,7 @@ jobs: - name: update dependencies url run: | ./bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }} + ./bin/update-rootfs.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }} - name: create PR uses: peter-evans/create-pull-request@v5 diff --git a/Makefile b/Makefile index 1f77a92..9c47405 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ ifneq (,$(findstring arm64,$(ARCH))) FINCH_OS_IMAGE_URL := $(FINCH_OS_AARCH64_URL) FINCH_OS_DIGEST ?= $(FINCH_OS_AARCH64_DIGEST) HOMEBREW_PREFIX ?= /opt/homebrew + + # TODO: Use Finch rootfs in Finch on Windows testing + FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/aarch64/finch-rootfs-production-arm64-1690563031.tar.zst else ifneq (,$(findstring x86_64,$(ARCH))) LIMA_ARCH = x86_64 LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1689037160.tar.gz @@ -38,6 +41,9 @@ else ifneq (,$(findstring x86_64,$(ARCH))) FINCH_OS_IMAGE_URL := $(FINCH_OS_x86_URL) FINCH_OS_DIGEST ?= $(FINCH_OS_x86_DIGEST) HOMEBREW_PREFIX ?= /usr/local + + # TODO: Use Finch rootfs in Finch on Windows testing + FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/x86-64/finch-rootfs-production-amd64-1690563027.tar.zst endif FINCH_OS_IMAGE_LOCATION ?= $(OUTDIR)/os/$(FINCH_OS_BASENAME) diff --git a/bin/update-rootfs.sh b/bin/update-rootfs.sh new file mode 100755 index 0000000..e4ef487 --- /dev/null +++ b/bin/update-rootfs.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -euxo pipefail + +DEPENDENCY_CLOUDFRONT_URL="https://deps.runfinch.com/" +AARCH64_FILENAME_PATTERN="common/aarch64/finch-rootfs-production-arm64-[0-9].*\.tar.zst$" +AMD64_FILENAME_PATTERN="common/x86-64/finch-rootfs-production-amd64-[0-9].*\.tar.zst$" +PLATFORM="common" +AARCH64="aarch64" +X86_64="x86-64" + +while getopts d: flag +do + case "${flag}" in + d) dependency_bucket=${OPTARG};; + esac +done + +[[ -z "$dependency_bucket" ]] && { echo "Error: Dependency bucket not set"; exit 1; } + +aarch64Deps=$(aws s3 ls s3://${dependency_bucket}/${PLATFORM}/${AARCH64} | grep "$AARCH64_FILENAME_PATTERN" | sort | tail -n 1 | awk '{print $4}') + +[[ -z "$aarch64Deps" ]] && { echo "Error: aarch64 dependency not found"; exit 1; } + +amd64Deps=$(aws s3 ls s3://${dependency_bucket}/${PLATFORM}/${X86_64} | grep "$AMD64_FILENAME_PATTERN" | sort | tail -n 1 | awk '{print $4}') + +[[ -z "$amd64Deps" ]] && { echo "Error: x86_64 dependency not found"; exit 1; } + +sed -E -i.bak 's|^([[:blank:]]*FINCH_ROOTFS_URL[[:blank:]]*\?=[[:blank:]]*'${DEPENDENCY_CLOUDFRONT_URL}')('${AARCH64_FILENAME_PATTERN}')|\1'$aarch64Deps'|' Makefile +sed -E -i.bak 's|^([[:blank:]]*FINCH_ROOTFS_URL[[:blank:]]*\?=[[:blank:]]*'${DEPENDENCY_CLOUDFRONT_URL}')('${AMD64_FILENAME_PATTERN}')|\1'$amd64Deps'|' Makefile