Add orca/orca_recovery_workflow modules, update docs

Makefile

@@ -207,6 +207,158 @@ destroy-rds: rds-init
 				-auto-approve"
 	eval $$TF_CMD
 
+orca_recovery_workflow: orca_recovery_workflow-init
+	$(banner)
+	cd $@
+	if [ -f "secrets/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars"
+		echo "Found maturity-specific secrets: $$SECRETS_OPT"
+		echo "***************************************************************"
+	fi
+	if [ -f "variables/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars"
+		echo "Found maturity-specific variables: $$VARIABLES_OPT"
+		echo "***************************************************************"
+	fi
+	terraform apply \
+		$$SECRETS_OPT \
+		$$VARIABLES_OPT \
+		-input=false \
+		-auto-approve \
+		-no-color
+
+
+plan-orca_recovery_workflow: orca_recovery_workflow-init
+	$(banner)
+	cd orca_recovery_workflow
+	if [ -f "secrets/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars"
+		echo "Found maturity-specific secrets: $$SECRETS_OPT"
+		echo "***************************************************************"
+	fi
+	if [ -f "variables/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars"
+		echo "Found maturity-specific variables: $$VARIABLES_OPT"
+		echo "***************************************************************"
+	fi
+	terraform plan \
+		$$SECRETS_OPT \
+		$$VARIABLES_OPT \
+		-input=false \
+		-no-color
+
+
+destroy-orca_recovery_workflow: orca_recovery_workflow-init
+	$(banner)
+	cd orca_recovery_workflow
+	if [ -f "secrets/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars"
+		echo "Found maturity-specific secrets: $$SECRETS_OPT"
+		echo "***************************************************************"
+	fi
+	if [ -f "variables/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars"
+		echo "Found maturity-specific variables: $$VARIABLES_OPT"
+		echo "***************************************************************"
+	fi
+	terraform destroy \
+		$$SECRETS_OPT \
+		$$VARIABLES_OPT \
+		-input=false \
+		-auto-approve \
+		-no-color
+
+
+
+# ---------------------------
+orca: orca-init
+	$(banner)
+	cd $@
+	if [ -f "secrets/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars"
+		echo "Found maturity-specific secrets: $$SECRETS_OPT"
+		echo "***************************************************************"
+	fi
+	if [ -f "variables/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars"
+		echo "Found maturity-specific variables: $$VARIABLES_OPT"
+		echo "***************************************************************"
+	fi
+	terraform apply \
+		$$SECRETS_OPT \
+		$$VARIABLES_OPT \
+		-input=false \
+		-auto-approve \
+		-no-color
+
+# ---------------------------
+destroy-orca: orca-init
+	$(banner)
+	cd orca
+	if [ -f "secrets/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars"
+		echo "Found maturity-specific secrets: $$SECRETS_OPT"
+		echo "***************************************************************"
+	fi
+	if [ -f "variables/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars"
+		echo "Found maturity-specific variables: $$VARIABLES_OPT"
+		echo "***************************************************************"
+	fi
+	terraform destroy \
+		$$SECRETS_OPT \
+		$$VARIABLES_OPT \
+		-input=false \
+		-auto-approve \
+		-no-color
+
+# ---------------------------
+plan-orca: orca-init
+	$(banner)
+	cd orca
+	if [ -f "secrets/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export SECRETS_OPT="-var-file=secrets/${MATURITY}.tfvars"
+		echo "Found maturity-specific secrets: $$SECRETS_OPT"
+		echo "***************************************************************"
+	fi
+	if [ -f "variables/${MATURITY}.tfvars" ]
+	then
+		echo "***************************************************************"
+		export VARIABLES_OPT="-var-file=variables/${MATURITY}.tfvars"
+		echo "Found maturity-specific variables: $$VARIABLES_OPT"
+		echo "***************************************************************"
+	fi
+	terraform plan \
+		$$SECRETS_OPT \
+		$$VARIABLES_OPT \
+		-input=false \
+		-no-color
+
+
+
+
 # ---------------------------
 pcrs: workflows/providers/* workflows/collections/* workflows/rules/*
 	if [ -z ${cumulus_id_rsa+x} ];  then echo "Env Var \$cumulus_id_rsa is not set, using ~/.ssh/id_rsa"; fi

README.md

@@ -38,12 +38,23 @@ You can run tests inside of a Docker container:
 
 ## Organization
 
-The repository is organized into three Terraform modules:
+The repository is organized into the following Terraform modules:
+
+### Cumulus Core Modules
 
 * `daac`: Creates DAAC-specific resources necessary for running Cumulus
 * `cumulus`: Creates all runtime Cumulus resources that can then be used
   to run ingest workflows.
 * `workflows`: Creates a Cumulus workflow with a sample Python lambda.
+* `rds`: This module deploys the default [https://github.com/nasa/cumulus/tree/master/tf-modules/cumulus-rds-tf] (`terraform-aws-cumulus-rds` serverless module)
+
+### Optional Cumulus Ecosystem Component Modules:
+
+* `orca`: Creates an instance of the
+  [https://nasa.github.io/cumulus-orca/](operational cloud recovery archive)
+* `orca_recovery_workflow`: Using configuration information from the `cumulus`
+  and `orca` modules creates a default Cumulus workflow, that can be used with
+  Orca for granule recovery.
 
 To customize the deployment for your DAAC, you will need to update
 variables and settings in a few of the modules. Specifically:
@@ -84,6 +95,41 @@ committed to git. The `.gitignore` file will ignore them by default.
 DAAC-specific workflows, lambdas, and configuration will be deployed
 by this module. Most workflow development work will be done here.
 
+### orca module
+
+This module will deploy an instance of ORCA ([https://nasa.github.io/cumulus-orca/](Operational Cloud Recovery Archive)).   The module configuration roughly translates to the configuration documentation listed on the ORCA page by exposing all of the variables from that module.  
+
+To configure this module, you will need to customize `orca/variables/*.tfvars` and `orca/secrets/*.tfvars` with appropriate values for each environment you're deploying this module to.  There is an `example.tfvars` file in each folder as a template for the values that are required, for all possible variable options consult the `orca/variables.tf` variables file and/or the ORCA documentation as the majority of these are passed through directly to the ORCA terraform module. 
+
+If using this module, you will need to configure the `cumulus` module's `use_orca` variable to true.  This will cause the `cumulus` module to read the `orca` module outputs to configure Cumulus to use ORCA.   No other configuration is required for Cumulus to use ORCA if using this module.
+
+This module _must_ be deployed _after_ the `daac` and `rds` submodules as it requires information from those modules to deploy, and _before_ the `cumulus` module.
+
+The Makefile supports the following actions for this module:
+
+* orca         - Init and deploy all `orca` stack resources
+* plan-orca    - Init and run a `terraform plan` on the `orca` stack to show the
+  intended change-set
+* destroy-orca - Init, and then destroy existing `orca` module resources.
+  Please note this will *not* configure any values derived from this module's
+  remote state in the `cumulus` or `orca_recovery_workflow` modules
+
+### orca_recovery_workflow module
+
+This module will deploy a basic granule recovery workflow for use with Cumulus.
+It makes use of remote state data from the `cumulus` module and `orca` module
+and must be deployed after both.   The deployed `OrcaRecoveryAdapterWorkflow`
+can be used via Cumulus collection configuration or Bulk Granule actions to
+trigger a recovery for granules as needed. The Makefile supports the following
+actions for this module:
+
+* orca_recovery_workflow         - Init and deploy all `orca_recovery_workflow`
+  stack resources
+* plan-orca_recovery_workflow    - Init and run a `terraform plan` on the
+  `orca_recovery_workflow` stack to show the intended change-set
+* destroy-orca_recovery_workflow - Init, and then destroy existing
+  `orca_recovery_workflow` module resources.
+
 ## Deploying Cumulus
 
 See [CIRRUS-core README](https://github.com/asfadmin/CIRRUS-core/blob/master/README.md).

orca/data.tf

@@ -0,0 +1,52 @@
+## --------------------------
+## Database configuration
+## --------------------------
+
+## TODO - Decide if it's valueable to allow for an alternate cluster to the rds module cluster
+## OBDAAC implementation makes it YAGNI
+data "aws_secretsmanager_secret" "rds_admin_credentials" {
+  arn =  data.terraform_remote_state.rds.outputs.admin_db_login_secret_arn
+}
+
+data "aws_secretsmanager_secret_version" "rds_admin_credentials" {
+  secret_id = data.aws_secretsmanager_secret.rds_admin_credentials.id
+}
+
+
+## --------------------------
+## AWS configuration
+## --------------------------
+
+data "aws_region" "current" {}
+data "aws_caller_identity" "current" {}
+
+
+data "aws_subnets" "subnet_ids" {
+  filter {
+    name = "tag:Name"
+    values = ["Private application ${data.aws_region.current.name}a subnet",
+    "Private application ${data.aws_region.current.name}b subnet"]
+  }
+}
+
+data "aws_vpc" "application_vpcs" {
+  tags = {
+    Name = "Application VPC"
+  }
+}
+
+## --------------------------
+## Remote state configuration
+## --------------------------
+
+data "terraform_remote_state" "rds" {
+  backend   = "s3"
+  workspace = var.DEPLOY_NAME
+  config    = local.rds_remote_state_config
+}
+
+data "terraform_remote_state" "daac" {
+  backend   = "s3"
+  workspace = var.DEPLOY_NAME
+  config    = local.daac_remote_state_config
+}

orca/locals.tf

@@ -0,0 +1,23 @@
+locals {
+  default_tags = {
+    Deployment = local.prefix
+  }
+  ## TODO - These should probably be module outputs from Cirrus rather than convention
+  system_bucket = data.terraform_remote_state.daac.outputs.bucket_map.internal.name
+
+  prefix = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}"
+  rds_remote_state_config = {
+    bucket = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}-tf-state-${substr(data.aws_caller_identity.current.account_id, -4, 4)}"
+    key    = "rds/terraform.tfstate"
+    region = data.aws_region.current.name
+  }
+  daac_remote_state_config = {
+    bucket = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}-tf-state-${substr(data.aws_caller_identity.current.account_id, -4, 4)}"
+    key    = "daac/terraform.tfstate"
+    region = data.aws_region.current.name
+  }
+  rds_admin_login = jsondecode(data.aws_secretsmanager_secret_version.rds_admin_credentials.secret_string)
+  permissions_boundary_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/NGAPShRoleBoundary"
+  daac_bucket_map = data.terraform_remote_state.daac.outputs.bucket_map
+  merged_bucket_map = merge(local.daac_bucket_map, { for n in var.orca_buckets : n => { name = n, type = "orca"} })
+}

orca/orca.tf

@@ -0,0 +1,62 @@
+module "orca" {
+  source = "https://github.com/nasa/cumulus-orca/releases/download/v8.1.0/cumulus-orca-terraform.zip"
+  ## --------------------------
+  ## Cumulus Variables
+  ## --------------------------
+  ## REQUIRED
+
+  aws_region               = data.aws_region.current.name
+  buckets                  = local.merged_bucket_map
+  lambda_subnet_ids        = data.aws_subnets.subnet_ids.ids
+  permissions_boundary_arn = local.permissions_boundary_arn
+  prefix                   = local.prefix
+  system_bucket            = local.system_bucket
+  vpc_id                   = data.aws_vpc.application_vpcs.id
+
+  ## OPTIONAL
+
+  tags        = local.default_tags
+
+  ## --------------------------
+  ## ORCA Variables
+  ## --------------------------
+  ## REQUIRED
+
+  db_admin_password        = local.rds_admin_login.password
+  db_admin_username        = local.rds_admin_login.username
+  db_host_endpoint         = local.rds_admin_login.host
+  db_user_password         = var.orca_db_user_password
+  dlq_subscription_email   = var.orca_dlq_subscription_email
+  orca_default_bucket      = var.orca_default_bucket
+  orca_reports_bucket_name = var.orca_reports_bucket
+  rds_security_group_id    = data.terraform_remote_state.rds.outputs.rds_security_group_id
+  s3_access_key            = var.orca_s3_access_key
+  s3_secret_key            = var.orca_s3_secret_key
+
+  ## OPTIONAL
+
+  default_multipart_chunksize_mb                         = var.default_multipart_chunksize_mb
+  metadata_queue_message_retention_time_seconds          = var.metadata_queue_message_retention_time_seconds
+  orca_default_recovery_type                             = var.orca_default_recovery_type
+  orca_default_storage_class                             = var.orca_default_storage_class
+  orca_delete_old_reconcile_jobs_frequency_cron          = var.orca_delete_old_reconcile_jobs_frequency_cron
+  orca_ingest_lambda_memory_size                         = var.orca_ingest_lambda_memory_size
+  orca_ingest_lambda_timeout                             = var.orca_ingest_lambda_timeout
+  orca_internal_reconciliation_expiration_days           = var.orca_internal_reconciliation_expiration_days
+  orca_reconciliation_lambda_memory_size                 = var.orca_reconciliation_lambda_memory_size
+  orca_reconciliation_lambda_timeout                     = var.orca_reconciliation_lambda_timeout
+  orca_recovery_buckets                                  = var.orca_recovery_buckets
+  orca_recovery_complete_filter_prefix                   = var.orca_recovery_complete_filter_prefix
+  orca_recovery_expiration_days                          = var.orca_recovery_expiration_days
+  orca_recovery_lambda_memory_size                       = var.orca_recovery_lambda_memory_size
+  orca_recovery_lambda_timeout                           = var.orca_recovery_lambda_timeout
+  orca_recovery_retry_limit                              = var.orca_recovery_retry_limit
+  orca_recovery_retry_interval                           = var.orca_recovery_retry_interval
+  orca_recovery_retry_backoff                            = var.orca_recovery_retry_backoff
+  s3_inventory_queue_message_retention_time_seconds      = var.s3_inventory_queue_message_retention_time_seconds
+  s3_report_frequency                                    = var.s3_report_frequency
+  sqs_delay_time_seconds                                 = var.sqs_delay_time_seconds
+  sqs_maximum_message_size                               = var.sqs_maximum_message_size
+  staged_recovery_queue_message_retention_time_seconds   = var.staged_recovery_queue_message_retention_time_seconds
+  status_update_queue_message_retention_time_seconds     = var.status_update_queue_message_retention_time_seconds
+}

orca/orca_cumulus_internal_s3_policy.tpl

@@ -0,0 +1,13 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam::${elb_account_id}:root"
+            },
+            "Action": "s3:PutObject",
+            "Resource": "arn:aws:s3:::${prefix}-internal/${prefix}-lb-gql-a-logs/*"
+        }
+    ]
+}

orca/outputs.tf

@@ -0,0 +1,7 @@
+output "merged_bucket_map" {
+    value = local.merged_bucket_map
+}
+
+output "orca_module" {
+    value = module.orca
+}