Merge pull request #2825 from redpanda-data/jackie-plugin-uploader

[CI] Distribute `connect` as a rpk managed plugin

.github/actions/upload_managed_plugin/action.yml

@@ -0,0 +1,86 @@
+name: upload-managed-plugin
+description: Upload binaries as rpk managed plugin
+inputs:
+  aws_access_key_id:
+    description: For accessing S3 bucket
+    required: true
+  aws_secret_access_key:
+    description: For accessing S3 bucket
+    required: true
+  aws_region:
+    description: For accessing S3 bucket
+    required: true
+  aws_s3_bucket:
+    description: S3 bucket to use
+    required: true
+  artifacts_file:
+    description: Path to goreleaser artifacts.json
+    required: true
+  metadata_file:
+    description: Path to goreleaser artifacts.json
+    required: true
+  project_root_dir:
+    description: Root dir of goreleaser project
+    required: true
+  plugin_name:
+    description: Should match the goreleaser build id for the binary  E.g. "connect"
+    required: true
+  goos:
+    description: CSV list of target OS's to filter on
+    required: true
+  goarch:
+    description: CSV list of target arch's to filter on
+    required: true
+  repo_hostname:
+    description: RPK Plugins repo hostname. E.g. rpk-plugins.redpanda.com
+    required: true
+  dry_run:
+    description: Dry run means skipping writes to S3
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws_access_key_id }}
+        aws-secret-access-key: ${{ inputs.aws_secret_access_key }}
+        aws-region: ${{ inputs.aws_region }}
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: install deps
+      working-directory: resources/plugin_uploader
+      shell: bash
+      run: pip install -r requirements.txt
+
+    - name: Upload archives
+      working-directory: resources/plugin_uploader
+      shell: bash
+      run: |
+        DRY_RUN_FLAG=${{ inputs.dry_run && '--dry-run' || '' }}
+        ./plugin_uploader.py upload-archives \
+          --artifacts-file=${{ inputs.artifacts_file }} \
+          --metadata-file=${{ inputs.metadata_file }} \
+          --project-root-dir=${{ inputs.project_root_dir }} \
+          --region=${{ inputs.aws_region }} \
+          --bucket=${{ inputs.aws_s3_bucket }} \
+          --plugin=${{ inputs.plugin_name }} \
+          --goos=${{ inputs.goos }} \
+          --goarch=${{ inputs.goarch }} \
+          $DRY_RUN_FLAG
+
+    - name: Upload manifest
+      working-directory: resources/plugin_uploader
+      shell: bash
+      run: |
+        DRY_RUN_FLAG=${{ inputs.dry_run && '--dry-run' || '' }}
+        ./plugin_uploader.py upload-manifest \
+          --region=${{ inputs.aws_region }} \
+          --bucket=${{ inputs.aws_s3_bucket }} \
+          --plugin=${{ inputs.plugin_name }} \
+          --repo-hostname=${{ inputs.repo_hostname }} \
+          $DRY_RUN_FLAG

.github/workflows/test_plugin_uploader.yml

@@ -0,0 +1,61 @@
+name: Test Plugin Uploader
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'resources/plugin_uploader/**'
+      - '.github/workflows/test_plugin_uploader.yml'
+  pull_request:
+    paths:
+      - 'resources/plugin_uploader/**'
+      - '.github/workflows/test_plugin_uploader.yml'
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - working-directory: resources/plugin_uploader
+        run: pip install -r requirements_test.txt
+
+      - working-directory: resources/plugin_uploader
+        run: pytest -vv .
+
+  ruff-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Lint with Ruff
+        working-directory: resources/plugin_uploader
+        run: |
+          pip install ruff==0.4.10
+          ruff check --output-format=github
+
+  pyright-type-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - working-directory: resources/plugin_uploader
+        run: pip install -r requirements_test.txt
+
+      - run: pip install pyright==1.1.378
+
+      - working-directory: resources/plugin_uploader
+        run: pyright

.github/workflows/upload_plugin.yml

@@ -0,0 +1,83 @@
+name: Upload rpk connect plugin to S3
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      # All runs triggered by tag will really push to S3.
+      # Take care when adding more patterns here.
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+'
+  pull_request:
+    # Keep CI snappy for unrelated PRs
+    paths:
+      - 'resources/plugin_uploader/**'
+      - '.github/workflows/upload_plugin.yml'
+      - '.github/actions/upload_managed_plugin/**'
+      - '.goreleaser.yml'
+  workflow_dispatch: {}
+
+env:
+  # Do dry run in most cases, UNLESS the triggering event was a "tag".
+  DRY_RUN: ${{ ! github.ref_type == 'tag' }}
+
+jobs:
+  upload_rpk_connect_plugin:
+    # Let's make this fast by using a beefy runner.
+    runs-on: ubuntu-latest-32
+    if: ${{ github.repository == 'redpanda-data/connect' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'redpanda-data/connect') }}
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.RP_AWS_CRED_REGION }}
+          role-to-assume: arn:aws:iam::${{ secrets.RP_AWS_CRED_ACCOUNT_ID }}:role/${{ vars.RP_AWS_CRED_BASE_ROLE_NAME }}${{ github.event.repository.name }}
+
+      - name: Get secrets from AWS Secrets Manager (for read/writing S3-backed plugins repo)
+        uses: aws-actions/aws-secretsmanager-get-secrets@v2
+        with:
+          secret-ids: |
+            ,sdlc/prod/github/rpk_plugin_publisher
+          parse-json-secrets: true
+
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 1.22.x
+          check-latest: true
+
+      - name: Build binaries (dry run / snapshot mode)
+        if: ${{ env.DRY_RUN }}
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          version: 1.26.2
+          args: build --snapshot
+
+      - name: Build binaries
+        if: ${{ ! env.DRY_RUN }}
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          version: 1.26.2
+          args: build
+
+      - name: Upload connect plugin to S3
+        uses: ./.github/actions/upload_managed_plugin
+        with:
+          aws_access_key_id: ${{ env.RPK_PLUGIN_PUBLISHER_AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ env.RPK_PLUGIN_PUBLISHER_AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_s3_bucket: "rpk-plugins-repo"
+          project_root_dir: ${{ github.workspace }}
+          artifacts_file: ${{ github.workspace }}/target/dist/artifacts.json
+          metadata_file: ${{ github.workspace }}/target/dist/metadata.json
+          plugin_name: "connect"
+          goos: linux,darwin
+          goarch: amd64,arm64
+          repo_hostname: rpk-plugins.redpanda.com
+          dry_run: ${{ env.DRY_RUN }}
+

.gitignore

@@ -9,3 +9,4 @@ release_notes.md
 .idea
 .vscode
 .op
+__pycache__

resources/plugin_uploader/README.md

@@ -0,0 +1,102 @@
+# Plugin uploader
+
+## Description
+
+```
+Usage: plugin_uploader.py [OPTIONS] COMMAND [ARGS]...
+
+CLI tool to upload/index goreleaser-built binaries to/in S3.
+
+Options:
+--help  Show this message and exit.
+
+Commands:
+upload-archives  Create tar.gz archives from binaries and upload to S3
+upload-manifest  Create manifest.json and upload to S3
+
+`plugin_uploader.py` is used to upload the binaries generated by goreleaser to S3 in a manner that is consumable by RPK as a plugin.
+
+```
+
+## Install
+
+`pip install -r requirements.txt`
+
+## How to use
+
+Primary use case is in GitHub Actions in response to creation of a GitHub release.
+
+See `.github/workflows/upload_plugin.yml` to see this in action.
+
+It's expected that you have used goreleaser to build a set of binaries for a given release tag (such as following a
+GitHub release tag creation).
+
+Goreleaser creates a `$DIST` directory (`dist/` by default) at the project root dir containing all built binaries and
+two JSON files:
+
+* `$DIST/<build-name>-<os>-<arch>/<binary-filename>`
+* ...
+* `$DIST/artifacts.json`
+* `$DIST/metadata.json`
+
+### Create archives from binaries and upload them
+
+Locate the `artifact.json` and `metadata.json` files produced by Goreleaser.
+E.g. `$DIST/artifacts.json`, `$DIST/metadata.json`.
+
+```shell
+./plugin_uploader.py upload-archives \
+                        --artifacts-file=$DIST/artifacts.json \
+                        --metadata-file=$DIST/metadata.json \
+                        --project-root-dir=<PROJECT_ROOT> \
+                        --region=<AWS_REGION> \
+                        --bucket=<AWS_S3_BUCKET> \
+                        --plugin=<PLUGIN_NAME> \
+                        --goos=<OS1,OS2,...> \
+                        --goarch=<ARCH1,ARCH2,...>
+```
+
+`PROJECT_ROOT` should be the root directory of the Golang project (by default, where `.goreleaser.yml` lives)
+
+`PLUGIN_NAME` should match the `<build-id>` as defined in goreleaser configs.
+
+It's assumed that the output binary filename is `redpanda-<build-id>`. E.g. for the **connect** project:
+
+* `build-id` is `connect`
+* Binary is `redpanda-connect`
+
+A binary is included for archival / upload only if it matches some `--goos` AND some `--goarch`.
+
+`--dry-run` is available for skipping final S3 upload step.
+
+AWS permissions are needed for these actions on the S3 bucket:
+
+* `s3:PutObject`
+* `s3:PutObjectTagging`
+  You may also need permissions on any AWS KMS keys used for server side encryption of the S3 bucket.
+
+### Create manifest.json and upload it
+
+This lists all archives for the specific plugin and constructs a `manifest.json` from the listing.
+
+This should be run after uploading any archives.
+
+```shell
+./plugin_uploader.py upload-manifest \
+                        --region=<AWS_REGION> \
+                        --bucket=<AWS_S3_BUCKET> \
+                        --plugin=<PLUGIN_NAME> \
+                        --repo-hostname=<REPO_HOSTNAME>
+```
+
+`--repo-hostname` is used for generating the right public facing download URLs for archives in the plugin repo. E.g.
+`rpk-plugins.redpanda.com`
+
+`--dry-run` is available for skipping the final S3 upload step.
+
+AWS permissions are needed for these actions on the S3 bucket:
+
+* `s3:PutObject`
+* `s3:ListBucket`
+* `s3:GetObjectTagging`
+  You may also need permissions on any AWS KMS keys used for server side encryption of the S3 bucket.