fix: DBTP-1383 - Set correct central log subscription filter destinat…

…ions (#233)

elasticache-redis/locals.tf

@@ -4,12 +4,15 @@ locals {
     "7.0" = "redis7"
     "6.2" = "redis6.x"
   }
-  central_log_destination_arn = "arn:aws:logs:eu-west-2:812359060647:destination:cwl_log_destination"
+
   tags = {
     application         = var.application
     environment         = var.environment
     managed-by          = "DBT Platform - Terraform"
     copilot-application = var.application
     copilot-environment = var.environment
   }
+
+  central_log_group_arns        = jsondecode(data.aws_ssm_parameter.log-destination-arn.value)
+  central_log_group_destination = var.environment == "prod" ? local.central_log_group_arns["prod"] : local.central_log_group_arns["dev"]
 }

elasticache-redis/main.tf

@@ -221,19 +221,22 @@ resource "aws_cloudwatch_log_group" "redis-engine-log-group" {
   kms_key_id        = aws_kms_key.redis-log-group-kms-key.arn
 }
 
+data "aws_ssm_parameter" "log-destination-arn" {
+  name = "/copilot/tools/central_log_groups"
+}
+
 resource "aws_cloudwatch_log_subscription_filter" "redis-subscription-filter-engine" {
   name            = "/aws/elasticache/${var.name}/${var.environment}/engine"
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = aws_cloudwatch_log_group.redis-engine-log-group.name
   filter_pattern  = ""
-  destination_arn = local.central_log_destination_arn
+  destination_arn = local.central_log_group_destination
 }
 
 resource "aws_cloudwatch_log_subscription_filter" "redis-subscription-filter-slow" {
   name            = "/aws/elasticache/${var.name}/${var.environment}/slow"
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = aws_cloudwatch_log_group.redis-slow-log-group.name
   filter_pattern  = ""
-  destination_arn = local.central_log_destination_arn
+  destination_arn = local.central_log_group_destination
 }
-

elasticache-redis/tests/unit.tftest.hcl

@@ -31,6 +31,13 @@ override_data {
   }
 }
 
+override_data {
+  target = data.aws_ssm_parameter.log-destination-arn
+  values = {
+    value = "{\"prod\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_prod\", \"dev\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev\"}"
+  }
+}
+
 run "aws_elasticache_replication_group_unit_test" {
   command = plan
 
@@ -348,7 +355,7 @@ run "aws_cloudwatch_log_subscription_filter_unit_test" {
   }
 
   assert {
-    condition     = aws_cloudwatch_log_subscription_filter.redis-subscription-filter-engine.destination_arn == "arn:aws:logs:eu-west-2:812359060647:destination:cwl_log_destination"
+    condition     = aws_cloudwatch_log_subscription_filter.redis-subscription-filter-engine.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
     error_message = "Invalid config for aws_cloudwatch_log_subscription_filter destination_arn"
   }
 
@@ -369,7 +376,7 @@ run "aws_cloudwatch_log_subscription_filter_unit_test" {
   }
 
   assert {
-    condition     = aws_cloudwatch_log_subscription_filter.redis-subscription-filter-slow.destination_arn == "arn:aws:logs:eu-west-2:812359060647:destination:cwl_log_destination"
+    condition     = aws_cloudwatch_log_subscription_filter.redis-subscription-filter-slow.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
     error_message = "Invalid config for aws_cloudwatch_log_subscription_filter destination_arn"
   }
 
@@ -384,6 +391,24 @@ run "aws_cloudwatch_log_subscription_filter_unit_test" {
   }
 }
 
+run "aws_cloudwatch_log_subscription_filter_destination_prod_unit_test" {
+  command = plan
+
+  variables {
+    environment = "prod"
+  }
+
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.redis-subscription-filter-engine.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_prod"
+    error_message = "Invalid config for aws_cloudwatch_log_subscription_filter destination_arn"
+  }
+
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.redis-subscription-filter-slow.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_prod"
+    error_message = "Invalid config for aws_cloudwatch_log_subscription_filter destination_arn"
+  }
+}
+
 run "test_create_conduit_iam_role" {
   command = plan
 

environment-pipelines/iam.tf

@@ -424,6 +424,10 @@ data "aws_iam_policy_document" "cloudwatch" {
   }
 }
 
+data "aws_ssm_parameter" "log-destination-arn" {
+  name = "/copilot/tools/central_log_groups"
+}
+
 data "aws_iam_policy_document" "logs" {
   statement {
     actions = [
@@ -442,7 +446,8 @@ data "aws_iam_policy_document" "logs" {
       "logs:PutSubscriptionFilter"
     ]
     resources = [
-      local.central_log_destination_arn
+      jsondecode(data.aws_ssm_parameter.log-destination-arn.value)["dev"],
+      jsondecode(data.aws_ssm_parameter.log-destination-arn.value)["prod"]
     ]
   }
 

environment-pipelines/locals.tf

@@ -137,6 +137,4 @@ locals {
 
   # Merge in the stage specific config from the stage_config.yml file:
   stages = [for stage in local.all_stages : merge(stage, local.stage_config[stage["type"]])]
-
-  central_log_destination_arn = "arn:aws:logs:eu-west-2:812359060647:destination:cwl_log_destination"
 }

environment-pipelines/tests/unit.tftest.hcl

@@ -203,6 +203,13 @@ override_data {
   }
 }
 
+override_data {
+  target = data.aws_ssm_parameter.log-destination-arn
+  values = {
+    value = "{\"prod\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_prod\", \"dev\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev\"}"
+  }
+}
+
 variables {
   application   = "my-app"
   repository    = "my-repository"

opensearch/cloudwatch.tf

@@ -1,4 +1,4 @@
-data "aws_ssm_parameter" "destination-arn" {
+data "aws_ssm_parameter" "log-destination-arn" {
   name = "/copilot/tools/central_log_groups"
 }
 
@@ -7,7 +7,7 @@ resource "aws_cloudwatch_log_subscription_filter" "opensearch_log_group_index_sl
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = "/aws/opensearch/${local.domain_name}/index-slow"
   filter_pattern  = ""
-  destination_arn = jsondecode(data.aws_ssm_parameter.destination-arn.value)["prod"]
+  destination_arn = local.central_log_group_destination
 
   depends_on = [aws_cloudwatch_log_group.opensearch_log_group_index_slow_logs]
 }
@@ -17,7 +17,7 @@ resource "aws_cloudwatch_log_subscription_filter" "opensearch_log_group_search_s
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = "/aws/opensearch/${local.domain_name}/search-slow"
   filter_pattern  = ""
-  destination_arn = jsondecode(data.aws_ssm_parameter.destination-arn.value)["prod"]
+  destination_arn = local.central_log_group_destination
 
   depends_on = [aws_cloudwatch_log_group.opensearch_log_group_search_slow_logs]
 }
@@ -27,7 +27,7 @@ resource "aws_cloudwatch_log_subscription_filter" "opensearch_log_group_es_appli
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = "/aws/opensearch/${local.domain_name}/es-application"
   filter_pattern  = ""
-  destination_arn = jsondecode(data.aws_ssm_parameter.destination-arn.value)["prod"]
+  destination_arn = local.central_log_group_destination
 
   depends_on = [aws_cloudwatch_log_group.opensearch_log_group_es_application_logs]
 }
@@ -37,7 +37,7 @@ resource "aws_cloudwatch_log_subscription_filter" "opensearch_log_group_audit_lo
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = "/aws/opensearch/${local.domain_name}/audit"
   filter_pattern  = ""
-  destination_arn = jsondecode(data.aws_ssm_parameter.destination-arn.value)["prod"]
+  destination_arn = local.central_log_group_destination
 
   depends_on = [aws_cloudwatch_log_group.opensearch_log_group_audit_logs]
 }

opensearch/locals.tf

@@ -25,4 +25,7 @@ locals {
 
   auto_tune_desired_state       = startswith(var.config.instance, "t2") || startswith(var.config.instance, "t3") ? "DISABLED" : "ENABLED"
   auto_tune_rollback_on_disable = startswith(var.config.instance, "t2") || startswith(var.config.instance, "t3") ? "DEFAULT_ROLLBACK" : "NO_ROLLBACK"
+
+  central_log_group_arns        = jsondecode(data.aws_ssm_parameter.log-destination-arn.value)
+  central_log_group_destination = var.environment == "prod" ? local.central_log_group_arns["prod"] : local.central_log_group_arns["dev"]
 }

opensearch/tests/unit.tftest.hcl

@@ -14,9 +14,9 @@ override_data {
 }
 
 override_data {
-  target = data.aws_ssm_parameter.destination-arn
+  target = data.aws_ssm_parameter.log-destination-arn
   values = {
-    value = "{\"prod\":\"arn:aws:ssm:eu-west-2:123456789987:parameter/copilot/tools/central_log_groups\"}"
+    value = "{\"prod\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_prod\", \"dev\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev\"}"
   }
 }
 
@@ -337,21 +337,41 @@ run "test_create_cloudwatch_subscription_filters" {
     error_message = "Cloudwatch log subscription filter name for cloudwatch log 'opensearch_log_group_index_slow_logs'Should be: '/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_index_slow'"
   }
 
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_index_slow_logs.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
+    error_message = "Cloudwatch log subscription filter destination ARN for cloudwatch log 'opensearch_log_group_index_slow_logs'Should be: 'arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev'"
+  }
+
   assert {
     condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_search_slow_logs.name == "/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_search_slow"
     error_message = "Cloudwatch log subscription filter name for cloudwatch log 'opensearch_log_group_search_slow_logs'Should be: '/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_search_slow'"
   }
 
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_search_slow_logs.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
+    error_message = "Cloudwatch log subscription filter destination ARN for cloudwatch log 'opensearch_log_group_search_slow_logs'Should be: 'arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev'"
+  }
+
   assert {
     condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_es_application_logs.name == "/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_es_application"
     error_message = "Cloudwatch log subscription filter name for cloudwatch log 'opensearch_log_group_es_application_logs'Should be: '/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_es_application'"
   }
 
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_es_application_logs.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
+    error_message = "Cloudwatch log subscription filter destination ARN for cloudwatch log 'opensearch_log_group_es_application_logs'Should be: 'arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev'"
+  }
+
   assert {
     condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_audit_logs.name == "/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_audit"
     error_message = "Cloudwatch log subscription filter name for cloudwatch log 'opensearch_log_group_audit_logs'Should be: '/aws/opensearch/my_app/my_env/my_name/opensearch_log_group_audit'"
   }
 
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_audit_logs.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
+    error_message = "Cloudwatch log subscription filter destination ARN for cloudwatch log 'opensearch_log_group_audit_logs'Should be: 'arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev'"
+  }
+
   assert {
     condition     = aws_cloudwatch_log_subscription_filter.opensearch_log_group_index_slow_logs.log_group_name == "/aws/opensearch/my-env-my-name/index-slow"
     error_message = "Cloudwatch log subscription filter log group name for cloudwatch log 'opensearch_log_group_index_slow_logs'Should be: '/aws/opensearch/my-env-my-name/index-slow'"

postgres/cloudwatch.tf

@@ -1,7 +1,6 @@
-
 data "aws_caller_identity" "current" {}
 
-data "aws_ssm_parameter" "destination-arn" {
+data "aws_ssm_parameter" "log-destination-arn" {
   name = "/copilot/tools/central_log_groups"
 }
 
@@ -10,7 +9,7 @@ resource "aws_cloudwatch_log_subscription_filter" "rds" {
   role_arn        = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/CWLtoSubscriptionFilterRole"
   log_group_name  = "/aws/rds/instance/${local.name}/postgresql"
   filter_pattern  = ""
-  destination_arn = jsondecode(data.aws_ssm_parameter.destination-arn.value)["prod"]
+  destination_arn = local.central_log_group_destination
 
   depends_on = [aws_db_instance.default]
 }

postgres/locals.tf

@@ -34,4 +34,7 @@ locals {
   rds_master_secret_name       = "${local.secret_prefix}_RDS_MASTER_ARN"
   read_only_secret_name        = "${local.secret_prefix}_READ_ONLY_USER"
   application_user_secret_name = "${local.secret_prefix}_APPLICATION_USER"
+
+  central_log_group_arns        = jsondecode(data.aws_ssm_parameter.log-destination-arn.value)
+  central_log_group_destination = var.environment == "prod" ? local.central_log_group_arns["prod"] : local.central_log_group_arns["dev"]
 }

postgres/tests/unit.tftest.hcl

@@ -19,6 +19,13 @@ override_data {
   }
 }
 
+override_data {
+  target = data.aws_ssm_parameter.log-destination-arn
+  values = {
+    value = "{\"prod\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_prod\", \"dev\":\"arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev\"}"
+  }
+}
+
 variables {
   application = "test-application"
   environment = "test-environment"
@@ -428,6 +435,11 @@ run "aws_cloudwatch_log_rds_subscription_filter_unit_test" {
     condition     = aws_cloudwatch_log_subscription_filter.rds.distribution == "ByLogStream"
     error_message = "Should be: ByLogStream"
   }
+
+  assert {
+    condition     = aws_cloudwatch_log_subscription_filter.rds.destination_arn == "arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
+    error_message = "Should be: arn:aws:logs:eu-west-2:123456789987:destination:central_log_groups_dev"
+  }
 }
 
 run "aws_lambda_function_unit_test" {