Merge pull request #58 from NandosUK/cloud-run-job

feat(cloud-run-jobs): Add new module + tests to run serverless jobs

gcp/cloud-run-v2-job/README.md

@@ -0,0 +1,52 @@
+# Overview
+
+This updated Terraform module is designed for deploying jobs to Google Cloud Run, adapting the versatility and ease of deploying services to the specific needs of job execution in a serverless environment. It extends the capabilities of Google Cloud Run services to jobs, allowing for scheduled tasks, batch processing, and one-off executions with the same level of configurability and scalability. Deploy your Cloud Run jobs efficiently and manage them seamlessly within the Google Cloud ecosystem.
+
+## Features
+
+### Google Cloud Run Job
+
+- **Job Name and Location** : Customizable name and deployment region for the job.
+- **Service Account** : Specify a service account for executing the job with appropriate permissions.
+- **Execution Environment** : Control over the job's execution environment, including CPU and memory resources.
+- **Environment Variables and Secrets** : Support for dynamic environment variables and secrets to configure the job execution context.
+- **Retry Policy** : Configuration options for job retries upon failure.
+- **Scheduled Execution** : Ability to define schedules for job execution using Cloud Scheduler.
+- **Artifact Repository** : Define the artifact repository for the job container image.
+- **Alert Configuration** : Set up custom alerting for job execution status.
+
+### Cloud Build Trigger for Jobs
+
+- **Trigger Configuration** : Advanced configuration for Cloud Build triggers specific to job deployment, including environment variables and artifact repository settings.
+
+## New Variables
+
+- `var.project_id`: The ID of the project where the job will be deployed.
+- `var.name`: Unique name for the Cloud Run job.
+- `var.project_region`: Deployment region for the Cloud Run job.
+- `var.cloud_run_service_account`: The service account to use for executing the Cloud Run job.
+- `var.environment`: Specifies the job's environment (e.g., development, production).
+- `var.artifact_repository`: The Docker artifact repository for the job container image.
+- `var.repository_name`: Name of the repository where the job's source code is located.
+- `var.env_vars`: Environment variables to set for the job's execution environment.
+- `var.secrets`: List of secrets from Secret Manager to be made available to the job.
+- `var.timeout`: Maximum allowed time duration the job may be active before being terminated.
+- `var.cpu_limit`, `var.memory_limit`: Resource limits for the job's container.
+- `var.max_retries`: Maximum number of retries for failed job executions.
+- `var.alert_config`: Configuration settings for alerts based on job execution and status.
+
+### Job Execution and Retry Policy Variables
+
+- `var.schedule`: Cron schedule for job execution (if applicable).
+- `var.attempt_deadline`: The deadline for each attempt of the job execution.
+- `var.retry_policy`: Policy for retrying failed job executions.
+
+See code for the complete list of available variables.
+
+## Usage
+
+For a practical demonstration of using this module to deploy Cloud Run jobs, refer to the example Terraform script in the example folder.
+
+Example of use:
+
+[test/gcp/cloud-run-job-v2.tf](../../test/gcp/cloud-run-job-v2.tf)

gcp/cloud-run-v2-job/main.tf

@@ -0,0 +1,131 @@
+data "google_project" "current" {
+  project_id = var.project_id
+}
+
+locals {
+  job_name = "${var.name}-job"
+}
+
+resource "google_cloud_run_v2_job" "default" {
+  name     = local.job_name
+  location = var.project_region
+
+  template {
+    template {
+      timeout         = var.timeout
+      max_retries     = var.max_retries
+      service_account = var.cloud_run_service_account
+      containers {
+        image = var.image
+
+        resources {
+          limits = {
+            cpu    = var.cpu_limit
+            memory = var.memory_limit
+          }
+        }
+
+        # Conditional environment variables
+        dynamic "env" {
+          for_each = var.env_vars
+          content {
+            name  = env.key
+            value = env.value
+          }
+        }
+        dynamic "env" {
+          for_each = length(var.secrets) > 0 ? tomap({ for s in var.secrets : s => s }) : {}
+          content {
+            name = env.key
+            value_source {
+              secret_key_ref {
+                secret  = "projects/${var.project_id}/secrets/${env.key}"
+                version = "latest"
+              }
+            }
+          }
+        }
+      }
+
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      launch_stage,
+      template[0].template[0].containers[0].image,
+    ]
+  }
+}
+
+
+resource "google_project_iam_binding" "sa_run_invoke" {
+  count   = var.cloud_run_service_account == null ? 0 : 1
+  project = var.project_id
+  role    = "roles/run.invoker"
+  members = ["serviceAccount:${var.cloud_run_service_account}"]
+}
+
+
+# Cloud Build trigger configuration
+module "trigger_provision" {
+  count           = var.create_trigger == true ? 1 : 0
+  source          = "../cloud-cloudbuild-trigger"
+  name            = "service-${var.name}-job-provision"
+  repository_name = var.repository_name
+  location        = var.location
+  description     = "Provision ${var.name} Job (CI/CD)"
+  filename        = "${var.service_path}/cloudbuild.yaml"
+  include         = concat(["${var.service_path}/**"], var.dependencies)
+  exclude         = ["${var.service_path}/functions/**"]
+  environment     = var.environment
+
+  # Substitution variables for Cloud Build Trigger
+  substitutions = merge({
+    _STAGE                    = "provision"
+    _BUILD_ENV                = var.environment
+    _SERVICE_NAME             = var.name
+    _DOCKER_ARTIFACT_REGISTRY = var.artifact_repository
+    _SERVICE_PATH             = var.service_path
+    _LOCATION                 = var.project_region
+    _SERVICE_ACCOUNT          = var.cloud_run_service_account
+    },
+    var.trigger_substitutions
+  )
+}
+
+module "cloud_run_alerts" {
+  source                = "../cloud-alerts"
+  project_id            = var.project_id
+  service_name          = local.job_name
+  enabled               = var.alert_config.enabled
+  threshold_value       = var.alert_config.threshold_value
+  duration              = var.alert_config.duration
+  alignment_period      = var.alert_config.alignment_period
+  auto_close            = var.alert_config.auto_close
+  notification_channels = var.alert_config.notification_channels
+}
+
+resource "google_cloud_scheduler_job" "scheduler_job" {
+  count            = var.enable_scheduler ? 1 : 0
+  provider         = google-beta
+  name             = var.scheduler_job_name
+  description      = "Scheduled job for triggering Cloud Run job: ${local.job_name}"
+  schedule         = var.schedule
+  attempt_deadline = var.attempt_deadline
+  region           = var.project_region
+  project          = var.project_id
+
+  retry_config {
+    retry_count = var.retry_count
+  }
+
+  http_target {
+    http_method = var.http_method
+    uri         = "https://${var.project_region}-run.googleapis.com/apis/run.googleapis.com/v1/namespaces/${data.google_project.current.number}/jobs/${local.job_name}:run"
+
+    oauth_token {
+      service_account_email = var.cloud_run_service_account
+    }
+  }
+}

gcp/cloud-run-v2-job/outputs.tf

@@ -0,0 +1,9 @@
+output "job_id" {
+  description = "Job ID of the Cloud Run Job"
+  value       = google_cloud_run_v2_job.default.id
+}
+
+output "job_uid" {
+  description = "Server assigned unique identifier for the Execution. The value is a UUID4 string and guaranteed to remain unchanged until the resource is deleted."
+  value       = google_cloud_run_v2_job.default.uid
+}

gcp/cloud-run-v2-job/variables.tf

@@ -0,0 +1,169 @@
+variable "project_id" {
+  description = "The ID of the project in which the resource belongs."
+  type        = string
+}
+
+variable "name" {
+  description = "(Required) Name must be unique within a namespace, within a Cloud Run region. No spaces or special characters allowed."
+  type        = string
+  validation {
+    condition     = can(regex("^[a-zA-Z0-9-_]+$", var.name))
+    error_message = "The name can only contain alphanumeric characters, hyphens, and underscores."
+  }
+}
+
+variable "project_region" {
+  description = "(Required) The location of the cloud run instance, e.g., europe-west2."
+  type        = string
+}
+
+variable "cloud_run_service_account" {
+  description = "(Optional) The service account that will be used as role/invoker."
+  type        = string
+  default     = null
+}
+
+variable "environment" {
+  description = "Environment that can be preview, preprod, dev, or prod."
+  type        = string
+  validation {
+    condition     = contains(["preview", "preprod", "prod", "dev"], var.environment)
+    error_message = "The environment must be one of: preview, preprod, dev, or prod."
+  }
+}
+
+variable "artifact_repository" {
+  description = "Artifact repository to use for this service."
+  type        = string
+  default     = "europe-west2-docker.pkg.dev/mgt-build-56d2ff6b/nandos-central-docker"
+}
+
+variable "repository_name" {
+  description = "Repo name where the service is located (in GitHub)."
+  type        = string
+}
+
+variable "create_trigger" {
+  description = "Create a Cloud Build trigger for this service."
+  type        = bool
+  default     = true
+}
+
+variable "max_retries" {
+  description = "Maximum number of retries for failed jobs."
+  default     = 3
+}
+
+variable "env_vars" {
+  description = "Environment variables."
+  type        = map(string)
+  default     = {}
+}
+
+variable "image" {
+  description = "The container image to run."
+  type        = string
+  default     = "us-docker.pkg.dev/cloudrun/container/job:latest"
+}
+
+variable "secrets" {
+  description = "List of secret names from Secret Manager."
+  type        = list(string)
+  default     = []
+}
+
+variable "timeout" {
+  description = "Max allowed time duration the task may be active before the system will actively try to mark it failed and kill associated containers."
+  type        = string
+  default     = "1800s" # Default to 30 min
+}
+
+variable "memory_limit" {
+  description = "Memory limit for the Cloud Run container."
+  type        = string
+  default     = "1024Mi" # Default to 1Gi memory
+}
+
+variable "cpu_limit" {
+  description = "CPU limit for the Cloud Run container."
+  type        = string
+  default     = "1000m" # Default to 1 cpu
+}
+
+variable "service_path" {
+  description = "Location for the main code and where the cloudbuild.yaml exists, for example /services/myapi."
+  type        = string
+}
+
+variable "location" {
+  description = "Cloud Build trigger location. If not specified, the default location will be global."
+  type        = string
+  default     = null
+}
+variable "alert_config" {
+  description = "Configuration for alerts."
+  type = object({
+    enabled               = bool
+    threshold_value       = number
+    duration              = number
+    alignment_period      = number
+    auto_close            = number
+    notification_channels = list(string)
+  })
+  default = {
+    enabled               = true
+    threshold_value       = 10.0
+    duration              = 300
+    alignment_period      = 60
+    auto_close            = 86400
+    notification_channels = []
+  }
+}
+
+variable "trigger_substitutions" {
+  description = "Substitution variables for Cloud Build Trigger."
+  type        = map(string)
+  default     = {}
+}
+
+variable "dependencies" {
+  description = "A list of glob-format dependencies for the Cloud Build trigger."
+  type        = list(string)
+  default     = []
+}
+
+variable "enable_scheduler" {
+  description = "Boolean flag to enable or disable the creation of a Cloud Scheduler job for triggering the Cloud Run job."
+  type        = bool
+  default     = false
+}
+
+variable "schedule" {
+  description = "The schedule on which the job will be executed, in the Unix cron format. For example, '*/8 * * * *' to run every 8 minutes."
+  type        = string
+  default     = "*/8 * * * *"
+}
+
+variable "attempt_deadline" {
+  description = "The time limit for each attempt of the job. The job must complete within this timeframe or be retried if retries are enabled."
+  type        = string
+  default     = "360s"
+}
+
+variable "scheduler_job_name" {
+  description = "The name of the scheduler job, must be unique within the project."
+  type        = string
+  default     = "schedule-job"
+}
+
+variable "http_method" {
+  description = "The HTTP method to use for the scheduler job when calling the Cloud Run job. Typically 'POST'."
+  type        = string
+  default     = "POST"
+}
+
+variable "retry_count" {
+  description = "The number of times to retry the execution of the job if the previous execution fails."
+  type        = number
+  default     = 3
+}

gcp/cloud-run-v2/README.md

@@ -1,8 +1,6 @@
-# Updated Terraform Google Cloud Run Module README
+# Overview
 
-## Overview
-
-This updated Terraform module provides a comprehensive set of reusable configurations for deploying services to Google Cloud Run. It extends the previous version by incorporating new variables like Cloud Armor for security, advanced probe settings, and more. Deploy your Cloud Run services easily and make them available on `https://MY_SERVICE.api.nandos.dev`.
+This Terraform module provides a comprehensive set of reusable configurations for deploying services to Google Cloud Run service / api. It extends the previous version by incorporating new variables like Cloud Armor for security, advanced probe settings, and more. Deploy your Cloud Run services easily and make them available on `https://MY_SERVICE.api.nandos.dev`.
 
 ## Branching Strategy