Add dry_run and fix s3 backend merging behaviour

.github/workflows/build.yml

@@ -38,10 +38,10 @@ jobs:
       uses: actions/checkout@v3
     - name: Pull LocalStack Docker image
       run: docker pull localstack/localstack &
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v2
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Install dependencies
       run: make install
     - name: Run code linter

README.md

@@ -24,6 +24,7 @@ pip install terraform-local
 ## Configurations
 
 The following environment variables can be configured:
+* `DRY_RUN`: Generate the override file without invoking Terraform
 * `TF_CMD`: Terraform command to call (default: `terraform`)
 * `AWS_ENDPOINT_URL`: hostname and port of the target LocalStack instance
 * `LOCALSTACK_HOSTNAME`: __(Deprecated)__ host name of the target LocalStack instance
@@ -48,6 +49,7 @@ please refer to the man pages of `terraform --help`.
 
 ## Change Log
 
+* v0.18.0: Add `DRY_RUN` and patch S3 backend entrypoints
 * v0.17.1: Add `packaging` module to install requirements
 * v0.17.0: Add option to use new endpoints S3 backend options
 * v0.16.1: Update Setuptools to exclude tests during packaging

bin/tflocal

@@ -27,6 +27,7 @@ if os.path.isdir(os.path.join(PARENT_FOLDER, ".venv")):
 from localstack_client import config  # noqa: E402
 import hcl2  # noqa: E402
 
+DRY_RUN = str(os.environ.get("DRY_RUN")).strip().lower() in ["1", "true"]
 DEFAULT_REGION = "us-east-1"
 DEFAULT_ACCESS_KEY = "test"
 AWS_ENDPOINT_URL = os.environ.get("AWS_ENDPOINT_URL")
@@ -35,6 +36,7 @@ LOCALHOST_HOSTNAME = "localhost.localstack.cloud"
 S3_HOSTNAME = os.environ.get("S3_HOSTNAME") or f"s3.{LOCALHOST_HOSTNAME}"
 USE_EXEC = str(os.environ.get("USE_EXEC")).strip().lower() in ["1", "true"]
 TF_CMD = os.environ.get("TF_CMD") or "terraform"
+TF_PROXIED_CMDS = ("init", "plan", "apply", "destroy")
 LS_PROVIDERS_FILE = os.environ.get("LS_PROVIDERS_FILE") or "localstack_providers_override.tf"
 LOCALSTACK_HOSTNAME = urlparse(AWS_ENDPOINT_URL).hostname or os.environ.get("LOCALSTACK_HOSTNAME") or "localhost"
 EDGE_PORT = int(urlparse(AWS_ENDPOINT_URL).port or os.environ.get("EDGE_PORT") or 4566)
@@ -153,12 +155,15 @@ def create_provider_config_file(provider_aliases=None):
 
     # write temporary config file
     providers_file = get_providers_file_path()
-    if os.path.exists(providers_file):
-        msg = f"Providers override file {providers_file} already exists - please delete it first"
-        raise Exception(msg)
+    write_provider_config_file(providers_file, tf_config)
+
+    return providers_file
+
+
+def write_provider_config_file(providers_file, tf_config):
+    """Write provider config into file"""
     with open(providers_file, mode="w") as fp:
         fp.write(tf_config)
-    return providers_file
 
 
 def get_providers_file_path() -> str:
@@ -186,9 +191,12 @@ def determine_provider_aliases() -> list:
 
 def generate_s3_backend_config() -> str:
     """Generate an S3 `backend {..}` block with local endpoints, if configured"""
+    is_tf_legacy = not (TF_VERSION.major > 1 or (TF_VERSION.major == 1 and TF_VERSION.minor > 5))
     backend_config = None
     tf_files = parse_tf_files()
-    for obj in tf_files.values():
+    for filename, obj in tf_files.items():
+        if LS_PROVIDERS_FILE == filename:
+            continue
         tf_configs = ensure_list(obj.get("terraform", []))
         for tf_config in tf_configs:
             backend_config = ensure_list(tf_config.get("backend"))
@@ -199,6 +207,13 @@ def generate_s3_backend_config() -> str:
     if not backend_config:
         return ""
 
+    legacy_endpoint_mappings = {
+        "endpoint": "s3",
+        "iam_endpoint": "iam",
+        "sts_endpoint": "sts",
+        "dynamodb_endpoint": "dynamodb",
+    }
+
     configs = {
         # note: default values, updated by `backend_config` further below...
         "bucket": "tf-test-state",
@@ -213,15 +228,28 @@ def generate_s3_backend_config() -> str:
             "dynamodb": get_service_endpoint("dynamodb"),
         },
     }
+    # Merge in legacy endpoint configs if not existing already
+    if is_tf_legacy and backend_config.get("endpoints"):
+        raise ValueError("Warning: Unsupported backend options detected (`endpoints`). Please make sure you always use the corresponding options to your Terraform version.")
+    for legacy_endpoint, endpoint in legacy_endpoint_mappings.items():
+        if legacy_endpoint in backend_config and (not backend_config.get("endpoints") or endpoint not in backend_config["endpoints"]):
+            if not backend_config.get("endpoints"):
+                backend_config["endpoints"] = {}
+            backend_config["endpoints"].update({endpoint: backend_config[legacy_endpoint]})
+    # Add any missing default endpoints
+    if backend_config.get("endpoints"):
+        backend_config["endpoints"] = {
+            k: (backend_config["endpoints"][k] if backend_config["endpoints"].get(k) else v)
+            for k, v in configs["endpoints"].items()}
     configs.update(backend_config)
-    get_or_create_bucket(configs["bucket"])
-    get_or_create_ddb_table(configs["dynamodb_table"], region=configs["region"])
+    if not DRY_RUN:
+        get_or_create_bucket(configs["bucket"])
+        get_or_create_ddb_table(configs["dynamodb_table"], region=configs["region"])
     result = TF_S3_BACKEND_CONFIG
     for key, value in configs.items():
         if isinstance(value, bool):
             value = str(value).lower()
         elif isinstance(value, dict):
-            is_tf_legacy = not (TF_VERSION.major > 1 or (TF_VERSION.major == 1 and TF_VERSION.minor > 5))
             if key == "endpoints" and is_tf_legacy:
                 value = textwrap.indent(
                     text=textwrap.dedent(f"""\
@@ -241,6 +269,25 @@ def generate_s3_backend_config() -> str:
     return result
 
 
+def check_override_file(providers_file):
+    """Checks override file existance"""
+    try:
+        if os.path.exists(providers_file):
+            msg = f"Providers override file {providers_file} already exists"
+            err_msg = msg + " - please delete it first"
+            if DRY_RUN:
+                msg += ". File will be overwritten."
+                print(msg)
+                print("\tOnly 'yes' will be accepted to approve.")
+                if input("\tEnter a value: ") != "yes":
+                    raise FileExistsError(err_msg)
+            else:
+                raise FileExistsError(err_msg)
+    except FileExistsError as e:
+        print(e)
+        exit(1)
+
+
 # ---
 # AWS CLIENT UTILS
 # ---
@@ -357,6 +404,11 @@ def get_or_create_ddb_table(table_name: str, region: str = None):
 # ---
 # TF UTILS
 # ---
+def is_override_needed(args) -> bool:
+    if any(map(lambda x: x in args, TF_PROXIED_CMDS)):
+        return True
+    return False
+
 
 def parse_tf_files() -> dict:
     """Parse the local *.tf files and return a dict of <filename> -> <resource_dict>"""
@@ -432,18 +484,26 @@ def main():
         print(f"Unable to determine version. See error message for details: {e}")
         exit(1)
 
-    # create TF provider config file
-    providers = determine_provider_aliases()
-    config_file = create_provider_config_file(providers)
+    if is_override_needed(sys.argv[1:]):
+        check_override_file(get_providers_file_path())
 
-    # call terraform command
-    try:
-        if USE_EXEC:
-            run_tf_exec(cmd, env)
-        else:
-            run_tf_subprocess(cmd, env)
-    finally:
-        os.remove(config_file)
+        # create TF provider config file
+        providers = determine_provider_aliases()
+        config_file = create_provider_config_file(providers)
+
+    # call terraform command if not dry-run or any of the commands
+    if not DRY_RUN or (DRY_RUN and not is_override_needed(sys.argv[1:])):
+        try:
+            if USE_EXEC:
+                run_tf_exec(cmd, env)
+            else:
+                run_tf_subprocess(cmd, env)
+        finally:
+            try:
+                os.remove(config_file)
+            except UnboundLocalError:
+                # fall through if haven't set during dry-run
+                pass
 
 
 if __name__ == "__main__":

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = terraform-local
-version = 0.17.1
+version = 0.18.0
 url = https://github.com/localstack/terraform-local
 author = LocalStack Team
 author_email = [email protected]
@@ -15,6 +15,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     License :: OSI Approved :: Apache Software License
     Topic :: Software Development :: Testing