MRG: Merge pull request #18 from octue/stream-events-to-bigquery

Stream events to BigQuery table
octue · Apr 15, 2024 · 83722f6 · 83722f6
2 parents 3e50952 + bd6b401
commit 83722f6
Show file tree

Hide file tree

Showing 7 changed files with 565 additions and 323 deletions.
diff --git a/.github/workflows/add-issues-to-octue-board.yml b/.github/workflows/add-issues-to-octue-board.yml
@@ -0,0 +1,15 @@
+name: add-issues-to-octue-board
+
+on:
+  issues:
+    types: [opened, reopened, transferred]
+
+jobs:
+  add-issues-to-octue-board:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add to Board
+        uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/octue/projects/22
+          github-token: ${{ secrets.OCTUE_PROJECT_ISSUES_TOKEN }}
diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml
@@ -2,7 +2,6 @@ name: cd
 
 on:
   push:
-    branches: main
 
   workflow_dispatch:
     inputs:
@@ -13,6 +12,7 @@ on:
 
 jobs:
   info:
+    if: "!contains(github.event.head_commit.message, 'skipci')"
     runs-on: ubuntu-latest
 
     timeout-minutes: 5
@@ -23,9 +23,9 @@ jobs:
     outputs:
       branch_tag_kebab: ${{ steps.get-deployment-info.outputs.branch_tag_kebab }}
       branch_tag_screaming: ${{ steps.get-deployment-info.outputs.branch_tag_screaming}}
-      image_latest_artefact: ${{ steps.get-deployment-info.outputs.image_latest_artefact}}
+      image_latest_artifact: ${{ steps.get-deployment-info.outputs.image_latest_artifact}}
       image_latest_tag: ${{ steps.get-deployment-info.outputs.image_latest_tag }}
-      image_version_artefact: ${{ steps.get-deployment-info.outputs.image_version_artefact}}
+      image_version_artifact: ${{ steps.get-deployment-info.outputs.image_version_artifact}}
       image_version_tag: ${{ steps.get-deployment-info.outputs.image_version_tag }}
       short_sha: ${{ steps.get-deployment-info.outputs.short_sha }}
       gcp_project_name: ${{ steps.get-deployment-info.outputs.gcp_project_name}}
@@ -34,25 +34,25 @@ jobs:
       gcp_resource_affix: ${{ steps.get-deployment-info.outputs.gcp_resource_affix}}
       gcp_service_name: ${{ steps.get-deployment-info.outputs.gcp_service_name}}
       version: ${{ steps.get-deployment-info.outputs.version }}
-      version_slug: ${{ steps.get-deployment-info.outputs.version_slug }}
+      revision_tag: ${{ steps.get-deployment-info.outputs.revision_tag }}
+      revision_tag_slug: ${{ steps.get-deployment-info.outputs.revision_tag_slug }}
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install poetry
         uses: snok/install-poetry@v1
 
       - name: Get deployment info
         id: get-deployment-info
-        uses: octue/get-deployment-info@0.1.2
+        uses: octue/get-deployment-info@0.3.2
         with:
           gcp_project_name: octue-sdk-python
           gcp_project_number: 437801218871
           gcp_region: europe-west1
           gcp_resource_affix: octue
           gcp_service_name: example-service-cloud-run
-          gcp_environment: main
 
   build:
     runs-on: ubuntu-latest
@@ -64,7 +64,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -74,15 +74,15 @@ jobs:
 
       - name: Authenticate with GCP Workload Identity
         id: auth
-        uses: google-github-actions/auth@v0
+        uses: google-github-actions/auth@v2
         with:
           # NOTE: If setting create_credentials_file=true, .dockerignore file must include `gha-creds-*.json` to avoid baking these credentials into build
           create_credentials_file: true
           workload_identity_provider: projects/${{ needs.info.outputs.gcp_project_number }}/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider
           service_account: github-actions@${{ needs.info.outputs.gcp_project_name }}.iam.gserviceaccount.com
 
       - name: Setup gcloud
-        uses: "google-github-actions/setup-gcloud@v0"
+        uses: "google-github-actions/setup-gcloud@v2"
 
       - name: Configure Docker for GCP
         run: gcloud auth configure-docker ${{ needs.info.outputs.gcp_region }}-docker.pkg.dev
@@ -105,32 +105,32 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           tags: |
-            ${{ needs.info.outputs.image_version_artefact}}
-            ${{ needs.info.outputs.image_latest_artefact}}
+            ${{ needs.info.outputs.image_version_artifact}}
+            ${{ needs.info.outputs.image_latest_artifact}}
 
       - name: Deploy to Cloud Run service
         id: deploy-service
-        uses: google-github-actions/deploy-cloudrun@v0
+        uses: google-github-actions/deploy-cloudrun@v2
         with:
           env_vars: |
             OCTUE_SERVICE_NAMESPACE=${{ needs.info.outputs.gcp_resource_affix }}
             OCTUE_SERVICE_NAME=${{ needs.info.outputs.gcp_service_name }}
-            OCTUE_SERVICE_REVISION_TAG=${{ needs.info.outputs.version_slug }}
+            OCTUE_SERVICE_REVISION_TAG=${{ needs.info.outputs.revision_tag }}
             COMPUTE_PROVIDER=GOOGLE_CLOUD_RUN
           service: ${{ needs.info.outputs.gcp_resource_affix }}-${{ needs.info.outputs.gcp_service_name }}
-          image: ${{ needs.info.outputs.image_version_artefact }}
+          image: ${{ needs.info.outputs.image_version_artifact }}
           region: ${{ needs.info.outputs.gcp_region }}
-          tag: v${{ needs.info.outputs.version_slug }}
+          tag: v${{ needs.info.outputs.revision_tag_slug }}
           flags: "--allow-unauthenticated"
 
       - name: Show deployed service URL
         run: echo "${{ steps.deploy-service.outputs.url }}"
 
-      - name: Create topic and subscription
-        uses: octue/create-push-subscription@0.3.0.beta-0
+      - name: Create service revision push subscription
+        uses: octue/create-push-subscription@0.4.0
         with:
           project_name: ${{ needs.info.outputs.gcp_project_name }}
           service_namespace: ${{ needs.info.outputs.gcp_resource_affix }}
           service_name: ${{ needs.info.outputs.gcp_service_name }}
-          service_revision_tag: ${{ needs.info.outputs.version_slug }}
+          service_revision_tag: ${{ needs.info.outputs.revision_tag }}
           push_endpoint: ${{ steps.deploy-service.outputs.url }}
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml
@@ -0,0 +1,14 @@
+name: semantic
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  check-semantic-version:
+    if: "!contains(github.event.head_commit.message, 'skipci')"
+    uses: octue/workflows/.github/workflows/check-semantic-version.yml@main
+    with:
+      path: pyproject.toml
+      breaking_change_indicated_by: minor
diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml
@@ -6,21 +6,13 @@
 
 name: update-pull-request
 
-on: pull_request
+on: [pull_request]
 
 jobs:
   description:
-    if: "!contains(github.event.pull_request.body, '<!--- SKIP AUTOGENERATED NOTES --->')"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: octue/[email protected]
-        id: pr-description
-        with:
-          pull_request_url: ${{ github.event.pull_request.url }}
-          api_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Update pull request body
-        uses: riskledger/update-pr-description@v2
-        with:
-          body: ${{ steps.pr-description.outputs.pull_request_description }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+    uses: octue/workflows/.github/workflows/generate-pull-request-description.yml@main
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+    permissions:
+      contents: read
+      pull-requests: write