From 7faf78693dd6036d2fd959f923b19b5bc2186187 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Wed, 11 Oct 2023 23:22:57 -0600 Subject: [PATCH 1/3] Add orca/orca_recovery_workflow modules, update docs --- Makefile | 152 ++++++++++++++ README.md | 48 ++++- orca/data.tf | 52 +++++ orca/locals.tf | 23 +++ orca/orca.tf | 59 ++++++ orca/orca_cumulus_internal_s3_policy.tpl | 13 ++ orca/outputs.tf | 7 + orca/policy.tf | 16 ++ orca/secrets/example.tfvars | 2 + orca/variables.tf | 185 ++++++++++++++++++ orca/variables/example.tfvars | 3 + orca/versions.tf | 22 +++ orca_recovery_workflow/data.tf | 13 ++ orca_recovery_workflow/locals.tf | 19 ++ .../orca_recovery_adapter_workflow.asl.json | 50 +++++ orca_recovery_workflow/recovery_workflow.tf | 17 ++ orca_recovery_workflow/variables.tf | 12 ++ orca_recovery_workflow/versions.tf | 22 +++ 18 files changed, 714 insertions(+), 1 deletion(-) create mode 100644 orca/data.tf create mode 100644 orca/locals.tf create mode 100644 orca/orca.tf create mode 100644 orca/orca_cumulus_internal_s3_policy.tpl create mode 100644 orca/outputs.tf create mode 100644 orca/policy.tf create mode 100644 orca/secrets/example.tfvars create mode 100644 orca/variables.tf create mode 100644 orca/variables/example.tfvars create mode 100644 orca/versions.tf create mode 100644 orca_recovery_workflow/data.tf create mode 100644 orca_recovery_workflow/locals.tf create mode 100644 orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json create mode 100644 orca_recovery_workflow/recovery_workflow.tf create mode 100644 orca_recovery_workflow/variables.tf create mode 100644 orca_recovery_workflow/versions.tf diff --git a/Makefile b/Makefile index e5c643f3..4069bd24 100644 --- a/Makefile +++ b/Makefile @@ -207,6 +207,158 @@ destroy-rds: rds-init -auto-approve" eval $$TF_CMD +orca_recovery_workflow: orca_recovery_workflow-init + $(banner) + cd $@ + if [ -f "secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + if [ -f "variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + terraform apply \ + $$SECRETS_OPT \ + $$VARIABLES_OPT \ + -input=false \ + -auto-approve \ + -no-color + + +plan-orca_recovery_workflow: orca_recovery_workflow-init + $(banner) + cd orca_recovery_workflow + if [ -f "secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + if [ -f "variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + terraform plan \ + $$SECRETS_OPT \ + $$VARIABLES_OPT \ + -input=false \ + -no-color + + +destroy-orca_recovery_workflow: orca_recovery_workflow-init + $(banner) + cd orca_recovery_workflow + if [ -f "secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + if [ -f "variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + terraform destroy \ + $$SECRETS_OPT \ + $$VARIABLES_OPT \ + -input=false \ + -auto-approve \ + -no-color + + + +# --------------------------- +orca: orca-init + $(banner) + cd $@ + if [ -f "secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + if [ -f "variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + terraform apply \ + $$SECRETS_OPT \ + $$VARIABLES_OPT \ + -input=false \ + -auto-approve \ + -no-color + +# --------------------------- +destroy-orca: orca-init + $(banner) + cd orca + if [ -f "secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + if [ -f "variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + terraform destroy \ + $$SECRETS_OPT \ + $$VARIABLES_OPT \ + -input=false \ + -auto-approve \ + -no-color + +# --------------------------- +plan-orca: orca-init + $(banner) + cd orca + if [ -f "secrets/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars" + echo "Found maturity-specific secrets: $$SECRETS_OPT" + echo "***************************************************************" + fi + if [ -f "variables/${MATURITY}.tfvars" ] + then + echo "***************************************************************" + export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars" + echo "Found maturity-specific variables: $$VARIABLES_OPT" + echo "***************************************************************" + fi + terraform plan \ + $$SECRETS_OPT \ + $$VARIABLES_OPT \ + -input=false \ + -no-color + + + + # --------------------------- pcrs: workflows/providers/* workflows/collections/* workflows/rules/* if [ -z ${cumulus_id_rsa+x} ]; then echo "Env Var \$cumulus_id_rsa is not set, using ~/.ssh/id_rsa"; fi diff --git a/README.md b/README.md index b3e9adea..c9e3a665 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,23 @@ You can run tests inside of a Docker container: ## Organization -The repository is organized into three Terraform modules: +The repository is organized into the following Terraform modules: + +### Cumulus Core Modules * `daac`: Creates DAAC-specific resources necessary for running Cumulus * `cumulus`: Creates all runtime Cumulus resources that can then be used to run ingest workflows. * `workflows`: Creates a Cumulus workflow with a sample Python lambda. +* `rds`: This module deploys the default [https://github.com/nasa/cumulus/tree/master/tf-modules/cumulus-rds-tf] (`terraform-aws-cumulus-rds` serverless module) + +### Optional Cumulus Ecosystem Component Modules: + +* `orca`: Creates an instance of the + [https://nasa.github.io/cumulus-orca/](operational cloud recovery archive) +* `orca_recovery_workflow`: Using configuration information from the `cumulus` + and `orca` modules creates a default Cumulus workflow, that can be used with + Orca for granule recovery. To customize the deployment for your DAAC, you will need to update variables and settings in a few of the modules. Specifically: @@ -84,6 +95,41 @@ committed to git. The `.gitignore` file will ignore them by default. DAAC-specific workflows, lambdas, and configuration will be deployed by this module. Most workflow development work will be done here. +### orca module + +This module will deploy an instance of ORCA ([https://nasa.github.io/cumulus-orca/](Operational Cloud Recovery Archive)). The module configuration roughly translates to the configuration documentation listed on the ORCA page by exposing all of the variables from that module. + +To configure this module, you will need to customize `orca/variables/*.tfvars` and `orca/secrets/*.tfvars` with appropriate values for each environment you're deploying this module to. There is an `example.tfvars` file in each folder as a template for the values that are required, for all possible variable options consult the `orca/variables.tf` variables file and/or the ORCA documentation as the majority of these are passed through directly to the ORCA terraform module. + +If using this module, you will need to configure the `cumulus` module's `use_orca` variable to true. This will cause the `cumulus` module to read the `orca` module outputs to configure Cumulus to use ORCA. No other configuration is required for Cumulus to use ORCA if using this module. + +This module _must_ be deployed _after_ the `daac` and `rds` submodules as it requires information from those modules to deploy, and _before_ the `cumulus` module. + +The Makefile supports the following actions for this module: + +* orca - Init and deploy all `orca` stack resources +* plan-orca - Init and run a `terraform plan` on the `orca` stack to show the + intended change-set +* destroy-orca - Init, and then destroy existing `orca` module resources. + Please note this will *not* configure any values derived from this module's + remote state in the `cumulus` or `orca_recovery_workflow` modules + +### orca_recovery_workflow module + +This module will deploy a basic granule recovery workflow for use with Cumulus. +It makes use of remote state data from the `cumulus` module and `orca` module +and must be deployed after both. The deployed `OrcaRecoveryAdapterWorkflow` +can be used via Cumulus collection configuration or Bulk Granule actions to +trigger a recovery for granules as needed. The Makefile supports the following +actions for this module: + +* orca_recovery_workflow - Init and deploy all `orca_recovery_workflow` + stack resources +* plan-orca_recovery_workflow - Init and run a `terraform plan` on the + `orca_recovery_workflow` stack to show the intended change-set +* destroy-orca_recovery_workflow - Init, and then destroy existing + `orca_recovery_workflow` module resources. + ## Deploying Cumulus See [CIRRUS-core README](https://github.com/asfadmin/CIRRUS-core/blob/master/README.md). diff --git a/orca/data.tf b/orca/data.tf new file mode 100644 index 00000000..9bc4e341 --- /dev/null +++ b/orca/data.tf @@ -0,0 +1,52 @@ +## -------------------------- +## Database configuration +## -------------------------- + +## TODO - Decide if it's valueable to allow for an alternate cluster to the rds module cluster +## OBDAAC implementation makes it YAGNI +data "aws_secretsmanager_secret" "rds_admin_credentials" { + arn = data.terraform_remote_state.rds.outputs.admin_db_login_secret_arn +} + +data "aws_secretsmanager_secret_version" "rds_admin_credentials" { + secret_id = data.aws_secretsmanager_secret.rds_admin_credentials.id +} + + +## -------------------------- +## AWS configuration +## -------------------------- + +data "aws_region" "current" {} +data "aws_caller_identity" "current" {} + + +data "aws_subnets" "subnet_ids" { + filter { + name = "tag:Name" + values = ["Private application ${data.aws_region.current.name}a subnet", + "Private application ${data.aws_region.current.name}b subnet"] + } +} + +data "aws_vpc" "application_vpcs" { + tags = { + Name = "Application VPC" + } +} + +## -------------------------- +## Remote state configuration +## -------------------------- + +data "terraform_remote_state" "rds" { + backend = "s3" + workspace = var.DEPLOY_NAME + config = local.rds_remote_state_config +} + +data "terraform_remote_state" "daac" { + backend = "s3" + workspace = var.DEPLOY_NAME + config = local.daac_remote_state_config +} diff --git a/orca/locals.tf b/orca/locals.tf new file mode 100644 index 00000000..6fb6687e --- /dev/null +++ b/orca/locals.tf @@ -0,0 +1,23 @@ +locals { + default_tags = { + Deployment = local.prefix + } + ## TODO - These should probably be module outputs from Cirrus rather than convention + system_bucket = data.terraform_remote_state.daac.outputs.bucket_map.internal.name + + prefix = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}" + rds_remote_state_config = { + bucket = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}-tf-state-${substr(data.aws_caller_identity.current.account_id, -4, 4)}" + key = "rds/terraform.tfstate" + region = data.aws_region.current.name + } + daac_remote_state_config = { + bucket = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}-tf-state-${substr(data.aws_caller_identity.current.account_id, -4, 4)}" + key = "daac/terraform.tfstate" + region = data.aws_region.current.name + } + rds_admin_login = jsondecode(data.aws_secretsmanager_secret_version.rds_admin_credentials.secret_string) + permissions_boundary_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/NGAPShRoleBoundary" + daac_bucket_map = data.terraform_remote_state.daac.outputs.bucket_map + merged_bucket_map = merge(local.daac_bucket_map, { for n in var.orca_buckets : n => { name = n, type = "orca"} }) +} \ No newline at end of file diff --git a/orca/orca.tf b/orca/orca.tf new file mode 100644 index 00000000..2daa727f --- /dev/null +++ b/orca/orca.tf @@ -0,0 +1,59 @@ +module "orca" { + source = "https://github.com/nasa/cumulus-orca/releases/download/v8.1.0/cumulus-orca-terraform.zip" + ## -------------------------- + ## Cumulus Variables + ## -------------------------- + ## REQUIRED + aws_region = data.aws_region.current.name + buckets = local.merged_bucket_map + lambda_subnet_ids = data.aws_subnets.subnet_ids.ids + permissions_boundary_arn = local.permissions_boundary_arn + prefix = local.prefix + system_bucket = local.system_bucket + vpc_id = data.aws_vpc.application_vpcs.id + + ## OPTIONAL + tags = local.default_tags + + ## -------------------------- + ## ORCA Variables + ## -------------------------- + ## REQUIRED + + db_admin_password = local.rds_admin_login.password + db_admin_username = local.rds_admin_login.username + db_host_endpoint = local.rds_admin_login.host + db_user_password = var.orca_db_user_password + dlq_subscription_email = var.orca_dlq_subscription_email + orca_default_bucket = var.orca_default_bucket + orca_reports_bucket_name = var.orca_reports_bucket + rds_security_group_id = data.terraform_remote_state.rds.outputs.rds_security_group_id + s3_access_key = var.orca_s3_access_key + s3_secret_key = var.orca_s3_secret_key + + ## OPTIONAL + default_multipart_chunksize_mb = var.default_multipart_chunksize_mb + metadata_queue_message_retention_time_seconds = var.metadata_queue_message_retention_time_seconds + orca_default_recovery_type = var.orca_default_recovery_type + orca_default_storage_class = var.orca_default_storage_class + orca_delete_old_reconcile_jobs_frequency_cron = var.orca_delete_old_reconcile_jobs_frequency_cron + orca_ingest_lambda_memory_size = var.orca_ingest_lambda_memory_size + orca_ingest_lambda_timeout = var.orca_ingest_lambda_timeout + orca_internal_reconciliation_expiration_days = var.orca_internal_reconciliation_expiration_days + orca_reconciliation_lambda_memory_size = var.orca_reconciliation_lambda_memory_size + orca_reconciliation_lambda_timeout = var.orca_reconciliation_lambda_timeout + orca_recovery_buckets = var.orca_recovery_buckets + orca_recovery_complete_filter_prefix = var.orca_recovery_complete_filter_prefix + orca_recovery_expiration_days = var.orca_recovery_expiration_days + orca_recovery_lambda_memory_size = var.orca_recovery_lambda_memory_size + orca_recovery_lambda_timeout = var.orca_recovery_lambda_timeout + orca_recovery_retry_limit = var.orca_recovery_retry_limit + orca_recovery_retry_interval = var.orca_recovery_retry_interval + orca_recovery_retry_backoff = var.orca_recovery_retry_backoff + s3_inventory_queue_message_retention_time_seconds = var.s3_inventory_queue_message_retention_time_seconds + s3_report_frequency = var.s3_report_frequency + sqs_delay_time_seconds = var.sqs_delay_time_seconds + sqs_maximum_message_size = var.sqs_maximum_message_size + staged_recovery_queue_message_retention_time_seconds = var.staged_recovery_queue_message_retention_time_seconds + status_update_queue_message_retention_time_seconds = var.status_update_queue_message_retention_time_seconds +} \ No newline at end of file diff --git a/orca/orca_cumulus_internal_s3_policy.tpl b/orca/orca_cumulus_internal_s3_policy.tpl new file mode 100644 index 00000000..ef9ced5c --- /dev/null +++ b/orca/orca_cumulus_internal_s3_policy.tpl @@ -0,0 +1,13 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::${elb_account_id}:root" + }, + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::${prefix}-internal/${prefix}-lb-gql-a-logs/*" + } + ] +} \ No newline at end of file diff --git a/orca/outputs.tf b/orca/outputs.tf new file mode 100644 index 00000000..44140d4a --- /dev/null +++ b/orca/outputs.tf @@ -0,0 +1,7 @@ +output "merged_bucket_map" { + value = local.merged_bucket_map +} + +output "orca_module" { + value = module.orca +} \ No newline at end of file diff --git a/orca/policy.tf b/orca/policy.tf new file mode 100644 index 00000000..a5f77b67 --- /dev/null +++ b/orca/policy.tf @@ -0,0 +1,16 @@ +## Bucket policy required for Orca load balancer +## See https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html + +resource "aws_s3_bucket_policy" "load_balancer_log_access" { + bucket = local.system_bucket + policy = data.template_file.load_balancer_s3_policy.rendered +} + +data "template_file" "load_balancer_s3_policy" { + template = file("${path.module}/orca_cumulus_internal_s3_policy.tpl") + + vars = { + prefix = local.prefix + elb_account_id = var.elb_account_id + } +} \ No newline at end of file diff --git a/orca/secrets/example.tfvars b/orca/secrets/example.tfvars new file mode 100644 index 00000000..05da0a11 --- /dev/null +++ b/orca/secrets/example.tfvars @@ -0,0 +1,2 @@ +orca_db_user_password="" +orca_default_bucket="" diff --git a/orca/variables.tf b/orca/variables.tf new file mode 100644 index 00000000..1f0b9d4a --- /dev/null +++ b/orca/variables.tf @@ -0,0 +1,185 @@ +## Required + +### Module + +variable "DEPLOY_NAME" { + type = string +} + +### Orca + +variable "orca_db_user_password" { + description = "Password for RDS Orca database user authentication" + type = string +} + +variable "orca_dlq_subscription_email" { + type = string + description = "The email to notify users when messages are received in dead letter SQS queue due to orca restore failure." + default = "test@email.com" ## TODO: Set this via secret and remove default +} + +variable "orca_default_bucket" { + type = string + description = "Default ORCA S3 Glacier bucket to use." +} + +variable "orca_reports_bucket" { + type = string + description = "ORCA Reports Bucket" +} + +variable "orca_buckets" { + type = list(string) +} + +## Optional + +### Module + +variable "MATURITY" { + type = string + default = "dev" +} + +variable "elb_account_id" { + type = string + default = "797873946194" +} + +### Orca + +# See available orca docs at https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-with-cumulus#creating-cumulus-tforcatf + +variable "default_multipart_chunksize_mb" { + type = number + default = 250 +} + + +variable "metadata_queue_message_retention_time_seconds" { + type = number + default = 777600 +} + +variable "orca_default_recovery_type" { + type = string + default = "Standard" +} + +variable "orca_default_storage_class" { + type = string + default = "GLACIER" +} + +variable "orca_delete_old_reconcile_jobs_frequency_cron" { + type = string + default = "cron(0 0 ? * SUN *)" +} + +variable "orca_ingest_lambda_memory_size" { + type = number + default = 2240 +} + +variable "orca_ingest_lambda_timeout" { + type = number + default = 600 +} + +variable "orca_internal_reconciliation_expiration_days" { + type = number + default = 30 +} + +variable "orca_reconciliation_lambda_memory_size" { + type = number + default = 128 +} + +variable "orca_reconciliation_lambda_timeout" { + type = number + default = 720 +} + +variable "orca_recovery_buckets" { + type = list(string) + default = [] +} + +variable "orca_recovery_complete_filter_prefix" { + type = string + default = "" +} + +variable "orca_recovery_expiration_days" { + type = number + default = 5 +} + +variable "orca_recovery_lambda_memory_size" { + type = number + default = 128 +} + +variable "orca_recovery_lambda_timeout" { + type = number + default = 720 +} + +variable "orca_recovery_retry_limit" { + type = number + default = 3 +} + +variable "orca_recovery_retry_interval" { + type = number + default = 1 +} + +variable "orca_recovery_retry_backoff" { + type = number + default = 2 +} + +variable "s3_inventory_queue_message_retention_time_seconds" { + type = number + default = 432000 +} + +variable "s3_report_frequency" { + type = string + default = "Daily" +} + +variable "sqs_delay_time_seconds" { + type = number + default = 0 +} + +variable "sqs_maximum_message_size" { + type = number + default = 262144 +} + +variable "staged_recovery_queue_message_retention_time_seconds" { + type = number + default = 432000 +} + +variable "status_update_queue_message_retention_time_seconds" { + type = number + default = 777600 +} + +variable "orca_s3_access_key" { + type = string + description = "Access key for communicating with Orca S3 buckets." + default = "" +} + +variable "orca_s3_secret_key" { + type = string + description = "Secret key for communicating with Orca S3 buckets." + default = "" +} \ No newline at end of file diff --git a/orca/variables/example.tfvars b/orca/variables/example.tfvars new file mode 100644 index 00000000..03719969 --- /dev/null +++ b/orca/variables/example.tfvars @@ -0,0 +1,3 @@ +orca_buckets = [""] +orca_default_bucket = "" +orca_reports_bucket = "" \ No newline at end of file diff --git a/orca/versions.tf b/orca/versions.tf new file mode 100644 index 00000000..25d5fdad --- /dev/null +++ b/orca/versions.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 0.13" +} +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + } + } + backend "s3" {} +} + +provider "aws" { + ignore_tags { + key_prefixes = ["gsfc-ngap"] + } +} diff --git a/orca_recovery_workflow/data.tf b/orca_recovery_workflow/data.tf new file mode 100644 index 00000000..3b8b527e --- /dev/null +++ b/orca_recovery_workflow/data.tf @@ -0,0 +1,13 @@ +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +data "terraform_remote_state" "daac" { + backend = "s3" + workspace = var.DEPLOY_NAME + config = local.daac_remote_state_config +} +data "terraform_remote_state" "cumulus" { + backend = "s3" + workspace = var.DEPLOY_NAME + config = local.cumulus_remote_state_config +} diff --git a/orca_recovery_workflow/locals.tf b/orca_recovery_workflow/locals.tf new file mode 100644 index 00000000..2429a105 --- /dev/null +++ b/orca_recovery_workflow/locals.tf @@ -0,0 +1,19 @@ +locals { + default_tags = { + Deployment = local.prefix + } + prefix = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}" + daac_remote_state_config = { + bucket = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}-tf-state-${substr(data.aws_caller_identity.current.account_id, -4, 4)}" + key = "daac/terraform.tfstate" + region = data.aws_region.current.name + } + cumulus_remote_state_config = { + bucket = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}-tf-state-${substr(data.aws_caller_identity.current.account_id, -4, 4)}" + key = "cumulus/terraform.tfstate" + region = data.aws_region.current.name + } + orca_recovery_adapter_task_arn = data.terraform_remote_state.cumulus.outputs.orca_recovery_adapter_task.task_arn + workflow_config = data.terraform_remote_state.cumulus.outputs.workflow_config + system_bucket = data.terraform_remote_state.daac.outputs.bucket_map.internal.name +} \ No newline at end of file diff --git a/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json b/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json new file mode 100644 index 00000000..6d5b3321 --- /dev/null +++ b/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json @@ -0,0 +1,50 @@ +{ + "Comment": "Orca Recovery Adapter", + "StartAt": "OrcaRecoveryAdapter", + "States": { + "OrcaRecoveryAdapter": { + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "buckets": "{$.meta.buckets}", + "fileBucketMaps": "{$.meta.collection.files}", + "asyncOperationId": "{$.cumulus_meta.asyncOperationId}", + "s3MultipartChunksizeMb": "{$.meta.collection.meta.s3MultipartChunksizeMb}", + "excludedFileExtensions": "{$.meta.collection.meta.orca.excludedFileExtensions}" + } + } + }, + "Type": "Task", + "Resource": "${orca_recovery_adapter_task}", + "Next": "WorkflowSucceeded", + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "WorkflowFailed", + "ResultPath": "$.exception" + } + ], + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "IntervalSeconds": 2, + "MaxAttempts": 3, + "BackoffRate": 2 + } + ] + }, + "WorkflowSucceeded": { + "Type": "Succeed" + }, + "WorkflowFailed": { + "Cause": "Workflow failed", + "Type": "Fail" + } + } + } + \ No newline at end of file diff --git a/orca_recovery_workflow/recovery_workflow.tf b/orca_recovery_workflow/recovery_workflow.tf new file mode 100644 index 00000000..c6ed9b4d --- /dev/null +++ b/orca_recovery_workflow/recovery_workflow.tf @@ -0,0 +1,17 @@ +module "orca_recovery_adapter_workflow" { + source = "https://github.com/nasa/cumulus/releases/download/v17.0.0/terraform-aws-cumulus-workflow.zip" + + prefix = local.prefix + name = "OrcaRecoveryAdapterWorkflow" + workflow_config = local.workflow_config + system_bucket = local.system_bucket + tags = local.default_tags + + state_machine_definition = templatefile( + "${path.module}/orca_recovery_adapter_workflow.asl.json", + { + orca_recovery_adapter_task: local.orca_recovery_adapter_task_arn + } + ) +} + diff --git a/orca_recovery_workflow/variables.tf b/orca_recovery_workflow/variables.tf new file mode 100644 index 00000000..f9e1fc27 --- /dev/null +++ b/orca_recovery_workflow/variables.tf @@ -0,0 +1,12 @@ +## Required + +variable "DEPLOY_NAME" { + type = string +} + +## Optional + +variable "MATURITY" { + type = string + default = "dev" +} \ No newline at end of file diff --git a/orca_recovery_workflow/versions.tf b/orca_recovery_workflow/versions.tf new file mode 100644 index 00000000..25d5fdad --- /dev/null +++ b/orca_recovery_workflow/versions.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 0.13" +} +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + } + } + backend "s3" {} +} + +provider "aws" { + ignore_tags { + key_prefixes = ["gsfc-ngap"] + } +} From 088f99432ab8357e6ab2b014b79cf9af1d06d1cd Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Wed, 11 Oct 2023 23:36:54 -0600 Subject: [PATCH 2/3] Minor formatting/edits --- orca/locals.tf | 2 +- orca/orca.tf | 5 ++++- orca/orca_cumulus_internal_s3_policy.tpl | 2 +- orca/outputs.tf | 2 +- orca/policy.tf | 2 +- orca/variables.tf | 2 +- orca/variables/example.tfvars | 2 +- orca_recovery_workflow/locals.tf | 2 +- .../orca_recovery_adapter_workflow.asl.json | 1 - orca_recovery_workflow/variables.tf | 2 +- 10 files changed, 12 insertions(+), 10 deletions(-) diff --git a/orca/locals.tf b/orca/locals.tf index 6fb6687e..83780721 100644 --- a/orca/locals.tf +++ b/orca/locals.tf @@ -20,4 +20,4 @@ locals { permissions_boundary_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/NGAPShRoleBoundary" daac_bucket_map = data.terraform_remote_state.daac.outputs.bucket_map merged_bucket_map = merge(local.daac_bucket_map, { for n in var.orca_buckets : n => { name = n, type = "orca"} }) -} \ No newline at end of file +} diff --git a/orca/orca.tf b/orca/orca.tf index 2daa727f..8a36eade 100644 --- a/orca/orca.tf +++ b/orca/orca.tf @@ -4,6 +4,7 @@ module "orca" { ## Cumulus Variables ## -------------------------- ## REQUIRED + aws_region = data.aws_region.current.name buckets = local.merged_bucket_map lambda_subnet_ids = data.aws_subnets.subnet_ids.ids @@ -13,6 +14,7 @@ module "orca" { vpc_id = data.aws_vpc.application_vpcs.id ## OPTIONAL + tags = local.default_tags ## -------------------------- @@ -32,6 +34,7 @@ module "orca" { s3_secret_key = var.orca_s3_secret_key ## OPTIONAL + default_multipart_chunksize_mb = var.default_multipart_chunksize_mb metadata_queue_message_retention_time_seconds = var.metadata_queue_message_retention_time_seconds orca_default_recovery_type = var.orca_default_recovery_type @@ -56,4 +59,4 @@ module "orca" { sqs_maximum_message_size = var.sqs_maximum_message_size staged_recovery_queue_message_retention_time_seconds = var.staged_recovery_queue_message_retention_time_seconds status_update_queue_message_retention_time_seconds = var.status_update_queue_message_retention_time_seconds -} \ No newline at end of file +} diff --git a/orca/orca_cumulus_internal_s3_policy.tpl b/orca/orca_cumulus_internal_s3_policy.tpl index ef9ced5c..aa4abe3d 100644 --- a/orca/orca_cumulus_internal_s3_policy.tpl +++ b/orca/orca_cumulus_internal_s3_policy.tpl @@ -10,4 +10,4 @@ "Resource": "arn:aws:s3:::${prefix}-internal/${prefix}-lb-gql-a-logs/*" } ] -} \ No newline at end of file +} diff --git a/orca/outputs.tf b/orca/outputs.tf index 44140d4a..8b12fd07 100644 --- a/orca/outputs.tf +++ b/orca/outputs.tf @@ -4,4 +4,4 @@ output "merged_bucket_map" { output "orca_module" { value = module.orca -} \ No newline at end of file +} diff --git a/orca/policy.tf b/orca/policy.tf index a5f77b67..6ef2287c 100644 --- a/orca/policy.tf +++ b/orca/policy.tf @@ -13,4 +13,4 @@ data "template_file" "load_balancer_s3_policy" { prefix = local.prefix elb_account_id = var.elb_account_id } -} \ No newline at end of file +} diff --git a/orca/variables.tf b/orca/variables.tf index 1f0b9d4a..77e65954 100644 --- a/orca/variables.tf +++ b/orca/variables.tf @@ -182,4 +182,4 @@ variable "orca_s3_secret_key" { type = string description = "Secret key for communicating with Orca S3 buckets." default = "" -} \ No newline at end of file +} diff --git a/orca/variables/example.tfvars b/orca/variables/example.tfvars index 03719969..40f5877f 100644 --- a/orca/variables/example.tfvars +++ b/orca/variables/example.tfvars @@ -1,3 +1,3 @@ orca_buckets = [""] orca_default_bucket = "" -orca_reports_bucket = "" \ No newline at end of file +orca_reports_bucket = "" diff --git a/orca_recovery_workflow/locals.tf b/orca_recovery_workflow/locals.tf index 2429a105..7c00308a 100644 --- a/orca_recovery_workflow/locals.tf +++ b/orca_recovery_workflow/locals.tf @@ -16,4 +16,4 @@ locals { orca_recovery_adapter_task_arn = data.terraform_remote_state.cumulus.outputs.orca_recovery_adapter_task.task_arn workflow_config = data.terraform_remote_state.cumulus.outputs.workflow_config system_bucket = data.terraform_remote_state.daac.outputs.bucket_map.internal.name -} \ No newline at end of file +} diff --git a/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json b/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json index 6d5b3321..77822460 100644 --- a/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json +++ b/orca_recovery_workflow/orca_recovery_adapter_workflow.asl.json @@ -47,4 +47,3 @@ } } } - \ No newline at end of file diff --git a/orca_recovery_workflow/variables.tf b/orca_recovery_workflow/variables.tf index f9e1fc27..952d8d57 100644 --- a/orca_recovery_workflow/variables.tf +++ b/orca_recovery_workflow/variables.tf @@ -9,4 +9,4 @@ variable "DEPLOY_NAME" { variable "MATURITY" { type = string default = "dev" -} \ No newline at end of file +} From b056ef3d24306bc58dc150680b843dc9a5505866 Mon Sep 17 00:00:00 2001 From: Jonathan Kovarik Date: Mon, 6 Nov 2023 09:54:29 -0700 Subject: [PATCH 3/3] Fix merge issue/respond to PR feedback --- orca/versions.tf | 3 --- orca_recovery_workflow/versions.tf | 3 --- 2 files changed, 6 deletions(-) diff --git a/orca/versions.tf b/orca/versions.tf index 25d5fdad..6786e21e 100644 --- a/orca/versions.tf +++ b/orca/versions.tf @@ -1,6 +1,3 @@ -terraform { - required_version = ">= 0.13" -} terraform { required_version = ">= 0.13" required_providers { diff --git a/orca_recovery_workflow/versions.tf b/orca_recovery_workflow/versions.tf index 25d5fdad..6786e21e 100644 --- a/orca_recovery_workflow/versions.tf +++ b/orca_recovery_workflow/versions.tf @@ -1,6 +1,3 @@ -terraform { - required_version = ">= 0.13" -} terraform { required_version = ">= 0.13" required_providers {