From e623ff63899ea190fb06e47c910aea4bbaabc315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20N=C3=A9meth?= Date: Fri, 20 Sep 2024 13:34:38 +0200 Subject: [PATCH] Introduce proper backend config merge (#61) --- README.md | 1 + bin/tflocal | 42 ++++++++++++++++++------------------- setup.cfg | 2 +- tests/test_apply.py | 50 ++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 6b7886c..7497285 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ please refer to the man pages of `terraform --help`. ## Change Log +* v0.20.0: Fix S3 backend option merging * v0.19.0: Add `SKIP_ALIASES` configuration environment variable * v0.18.2: Fix warning on aliased custom endpoint names * v0.18.1: Fix issue with not proxied commands diff --git a/bin/tflocal b/bin/tflocal index 5c57489..779c7d7 100755 --- a/bin/tflocal +++ b/bin/tflocal @@ -55,17 +55,7 @@ provider "aws" { """ TF_S3_BACKEND_CONFIG = """ terraform { - backend "s3" { - region = "" - bucket = "" - key = "" - dynamodb_table = "" - - access_key = "test" - secret_key = "test" - - skip_credentials_validation = true - skip_metadata_api_check = true + backend "s3" { } } """ @@ -265,6 +255,10 @@ def generate_s3_backend_config() -> str: "key": "terraform.tfstate", "dynamodb_table": "tf-test-state", "region": get_region(), + "skip_credentials_validation": True, + "skip_metadata_api_check": True, + "secret_key": "test", + "endpoints": { "s3": get_service_endpoint("s3"), "iam": get_service_endpoint("iam"), @@ -278,40 +272,44 @@ def generate_s3_backend_config() -> str: print("Warning: Unsupported backend option(s) detected (`endpoints`). Please make sure you always use the corresponding options to your Terraform version.") exit(1) for legacy_endpoint, endpoint in legacy_endpoint_mappings.items(): + if legacy_endpoint in backend_config and backend_config.get("endpoints") and endpoint in backend_config["endpoints"]: + del backend_config[legacy_endpoint] + continue if legacy_endpoint in backend_config and (not backend_config.get("endpoints") or endpoint not in backend_config["endpoints"]): if not backend_config.get("endpoints"): backend_config["endpoints"] = {} backend_config["endpoints"].update({endpoint: backend_config[legacy_endpoint]}) + del backend_config[legacy_endpoint] # Add any missing default endpoints if backend_config.get("endpoints"): backend_config["endpoints"] = { k: backend_config["endpoints"].get(k) or v for k, v in configs["endpoints"].items()} + backend_config["access_key"] = get_access_key(backend_config) if CUSTOMIZE_ACCESS_KEY else DEFAULT_ACCESS_KEY configs.update(backend_config) if not DRY_RUN: get_or_create_bucket(configs["bucket"]) get_or_create_ddb_table(configs["dynamodb_table"], region=configs["region"]) result = TF_S3_BACKEND_CONFIG - for key, value in configs.items(): + config_options = "" + for key, value in sorted(configs.items()): if isinstance(value, bool): value = str(value).lower() elif isinstance(value, dict): if key == "endpoints" and is_tf_legacy: - value = textwrap.indent( - text=textwrap.dedent(f"""\ - endpoint = "{value["s3"]}" - iam_endpoint = "{value["iam"]}" - sts_endpoint = "{value["sts"]}" - dynamodb_endpoint = "{value["dynamodb"]}" - """), - prefix=" " * 4) + for legacy_endpoint, endpoint in legacy_endpoint_mappings.items(): + config_options += f'\n {legacy_endpoint} = "{configs[key][endpoint]}"' + continue else: value = textwrap.indent( text=f"{key} = {{\n" + "\n".join([f' {k} = "{v}"' for k, v in value.items()]) + "\n}", prefix=" " * 4) + config_options += f"\n{value}" + continue else: - value = str(value) - result = result.replace(f"<{key}>", value) + value = f'"{str(value)}"' + config_options += f'\n {key} = {value}' + result = result.replace("", config_options) return result diff --git a/setup.cfg b/setup.cfg index a158c0a..34c600f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = terraform-local -version = 0.19.0 +version = 0.20.0 url = https://github.com/localstack/terraform-local author = LocalStack Team author_email = info@localstack.cloud diff --git a/tests/test_apply.py b/tests/test_apply.py index 238da97..2c44b2b 100644 --- a/tests/test_apply.py +++ b/tests/test_apply.py @@ -230,7 +230,7 @@ def test_dry_run(monkeypatch): override_file = os.path.join(temp_dir, "localstack_providers_override.tf") assert check_override_file_exists(override_file) - assert check_override_file_backend_content(override_file, is_legacy=is_legacy_tf) + assert check_override_file_backend_endpoints_content(override_file, is_legacy=is_legacy_tf) # assert that bucket with state file exists s3 = client("s3", region_name="us-east-2") @@ -276,6 +276,50 @@ def check_override_file_content(override_file): return True +def test_s3_backend_configs_merge(monkeypatch): + monkeypatch.setenv("DRY_RUN", "1") + state_bucket = "tf-state-conf-merge" + state_table = "tf-state-conf-merge" + # Temporarily change "." -> "-" as aws provider >5.55.0 fails with LocalStack + # by calling aws-global pseudo region at S3 bucket creation instead of us-east-1 + bucket_name = "bucket-conf-merge" + config = """ + terraform { + backend "s3" { + bucket = "%s" + key = "terraform.tfstate" + dynamodb_table = "%s" + region = "us-east-2" + skip_credentials_validation = true + encryption = true + use_path_style = true + acl = "bucket-owner-full-control" + } + } + resource "aws_s3_bucket" "test-bucket" { + bucket = "%s" + } + """ % (state_bucket, state_table, bucket_name) + temp_dir = deploy_tf_script(config, cleanup=False, user_input="yes") + override_file = os.path.join(temp_dir, "localstack_providers_override.tf") + assert check_override_file_exists(override_file) + assert check_override_file_backend_extra_content(override_file) + rmtree(temp_dir) + + +def check_override_file_backend_extra_content(override_file): + try: + with open(override_file, "r") as fp: + result = hcl2.load(fp) + result = result["terraform"][0]["backend"][0]["s3"] + except Exception as e: + raise Exception(f'Unable to parse "{override_file}" as HCL file: {e}') + + return result.get("use_path_style") is True and \ + result.get("encryption") is True and \ + result.get("acl") == "bucket-owner-full-control" + + @pytest.mark.parametrize("endpoints", [ '', 'endpoint = "http://s3-localhost.localstack.cloud:4566"', @@ -314,7 +358,7 @@ def test_s3_backend_endpoints_merge(monkeypatch, endpoints: str): temp_dir = deploy_tf_script(config, cleanup=False, user_input="yes") override_file = os.path.join(temp_dir, "localstack_providers_override.tf") assert check_override_file_exists(override_file) - assert check_override_file_backend_content(override_file, is_legacy=is_legacy_tf) + assert check_override_file_backend_endpoints_content(override_file, is_legacy=is_legacy_tf) rmtree(temp_dir) @@ -322,7 +366,7 @@ def check_override_file_exists(override_file): return os.path.isfile(override_file) -def check_override_file_backend_content(override_file, is_legacy: bool = False): +def check_override_file_backend_endpoints_content(override_file, is_legacy: bool = False): legacy_options = ( "endpoint", "iam_endpoint",