Merge pull request #54 from NandosUK/run-boost

New subscriptions to DI module

gcp/data-ingestor-subscription/README.md

@@ -0,0 +1,14 @@
+# Terraform Google Pub/Sub Subscription Module README
+
+## Overview
+
+This module creates a new Google Pub/Sub Subscription to a specific Topic from Nandos Data Ingestor.
+Make sure you add permissions to the relevant service accounts permission
+
+Example:
+
+https://github.com/NandosUK/data-ingestor/blob/main/services/schemas/tables/raw_restaurant_operations/restaurant_admin_api_events.ts#L7
+
+## Example usage:
+
+Check `test/gcp/data-ingestor-subscription.tf` for usage

gcp/data-ingestor-subscription/main.tf

@@ -0,0 +1,38 @@
+locals {
+  data_ingestor_project_id = var.environment == "dev" ? "preview-data-ingestor-c0edc062" : var.environment == "preview" ? "preview-data-ingestor-c0edc062" : var.environment == "preprod" ? "preprod-data-ingestor-6ee5b6e2" : var.environment == "prod" ? "prod-data-ingestor-40f9b4fb" : "preview-data-ingestor-c0edc062"
+  topic_name               = "projects/${local.data_ingestor_project_id}/topics/${var.topic_name}"
+}
+
+
+resource "google_pubsub_subscription" "pubsub_subscription" {
+  name                       = "${var.name}-subscription"
+  topic                      = local.topic_name
+  ack_deadline_seconds       = var.ack_deadline_seconds
+  message_retention_duration = var.message_retention_duration
+  retain_acked_messages      = var.retain_acked_messages
+  enable_message_ordering    = var.enable_message_ordering
+
+  labels = {
+    info        = "terraform-managed-subscription"
+    environment = var.environment
+    name        = var.name
+  }
+
+  dynamic "push_config" {
+    for_each = var.subscription_type == "push" ? [1] : []
+    content {
+      push_endpoint = var.push_endpoint
+
+      attributes = {
+        x-goog-version = "v1"
+        environment    = var.environment
+        service        = "data-ingestor"
+      }
+
+      oidc_token {
+        service_account_email = var.service_account_email
+        audience              = var.audience
+      }
+    }
+  }
+}

gcp/data-ingestor-subscription/outputs.tf

@@ -0,0 +1,3 @@
+output "pubsub_subscription_name" {
+  value = google_pubsub_subscription.pubsub_subscription.name
+}

gcp/data-ingestor-subscription/variables.tf

@@ -0,0 +1,73 @@
+variable "subscription_type" {
+  type        = string
+  description = "The type of the subscription to create, can be 'push' or 'pull'."
+  default     = "push"
+}
+
+variable "name" {
+  type        = string
+  description = "The name of the subscription, should be the same as your service"
+}
+variable "topic_name" {
+  type        = string
+  description = "Make sure this topic exists"
+}
+variable "environment" {
+  type        = string
+  description = "The environment that can be preview, preprod, dev or prod"
+  validation {
+    condition     = contains(["preview", "preprod", "prod", "dev"], var.environment)
+    error_message = "The environment must be one of: preview, preprod, dev or prod."
+  }
+}
+variable "service_account_email" {
+  type        = string
+  description = "The service account email to be used for the subscription"
+}
+variable "push_endpoint" {
+  type        = string
+  description = "The endpoint to push messages to could be a cloud run, cloud function, or any other HTTP endpoint"
+  default     = null
+}
+
+variable "audience" {
+  type        = string
+  description = "The audience to be used for the subscription push endpoint, usually the same as the push endpoint"
+  default     = null
+}
+
+variable "message_retention_duration" {
+  description = "How long to retain unacknowledged messages in the subscription's backlog, specified as a duration in seconds."
+  type        = string
+  default     = "1200s" # Default to 20 minutes
+}
+
+variable "retain_acked_messages" {
+  description = "Indicates whether to retain acknowledged messages."
+  type        = bool
+  default     = true
+}
+
+variable "ack_deadline_seconds" {
+  description = "The maximum time after a subscriber receives a message before the subscriber should acknowledge the message."
+  type        = number
+  default     = 20
+}
+
+variable "expiration_ttl" {
+  description = "Specifies the \"time-to-live\" duration for an associated resource. The resource expires if it is not active for a period of ttl."
+  type        = string
+  default     = "300000.5s" # Default to about 3.5 days
+}
+
+variable "minimum_backoff" {
+  description = "The minimum delay between consecutive deliveries of a given message."
+  type        = string
+  default     = "10s"
+}
+
+variable "enable_message_ordering" {
+  description = "If true, messages published with the same orderingKey in PubsubMessage will be delivered to the subscribers in the order in which they are received by the Pub/Sub system."
+  type        = bool
+  default     = false
+}

test/gcp/data-ingestor-subscription.tf

@@ -0,0 +1,26 @@
+module "pubsub_subscription_test_push" {
+  source                = "../../gcp/data-ingestor-subscription"
+  name                  = "example-push-subscription"
+  topic_name            = "example-topic"
+  environment           = "dev"
+  subscription_type     = "push"
+  service_account_email = "[email protected]"
+  push_endpoint         = "https://example-endpoint.com" # Required if subscription_type is "push"
+  audience              = "https://example-endpoint.com" # Required if subscription_type is "push"
+}
+
+module "pubsub_subscription_test_pull" {
+  source                     = "../../gcp/data-ingestor-subscription"
+  name                       = "example-pull-subscription"
+  topic_name                 = "example-topic"
+  environment                = "dev"
+  subscription_type          = "pull"
+  service_account_email      = "[email protected]"
+  message_retention_duration = "1800s" # 30 minutes
+  retain_acked_messages      = true
+  ack_deadline_seconds       = 30
+  expiration_ttl             = "604800s" # 7 days
+  minimum_backoff            = "15s"
+  enable_message_ordering    = false
+
+}