add initial simple version of aws-replicator config UI (#37)

.github/workflows/aws-replicator.yml

@@ -55,8 +55,8 @@ jobs:
           pip install awscli-local[ver1]
           pip install terraform-local
 
-          find  /home/runner/.cache/localstack/volume/lib/extensions/python_venv/lib/python3.10/site-packages/aws*
-          ls -la  /home/runner/.cache/localstack/volume/lib/extensions/python_venv/lib/python3.10/site-packages/aws*
+          find  /home/runner/.cache/localstack/volume/lib/extensions/python_venv/lib/python3.11/site-packages/aws*
+          ls -la  /home/runner/.cache/localstack/volume/lib/extensions/python_venv/lib/python3.11/site-packages/aws*
           DEBUG=1 localstack start -d
           localstack wait
 

aws-replicator/MANIFEST.in

@@ -0,0 +1,3 @@
+recursive-include aws_replicator *.html
+recursive-include aws_replicator *.js
+recursive-include aws_replicator *.png

aws-replicator/Makefile

@@ -2,6 +2,7 @@ VENV_BIN = python3 -m venv
 VENV_DIR ?= .venv
 VENV_ACTIVATE = $(VENV_DIR)/bin/activate
 VENV_RUN = . $(VENV_ACTIVATE)
+TEST_PATH ?= tests
 PIP_CMD ?= pip
 
 venv: $(VENV_ACTIVATE)
@@ -29,7 +30,7 @@ install: venv
 	$(VENV_RUN); $(PIP_CMD) install -e ".[test]"
 
 test: venv
-	$(VENV_RUN); python -m pytest tests
+	$(VENV_RUN); python -m pytest $(TEST_PATH)
 
 dist: venv
 	$(VENV_RUN); python setup.py sdist bdist_wheel

aws-replicator/README.md

@@ -30,6 +30,8 @@ $ localstack aws proxy -s dynamodb,s3,cognito-idp
 
 **Warning:** Be careful when using the proxy - make sure to _never_ give access to production accounts or any critical/sensitive data!
 
+**Note:** The replicator CLI currently works only when installing the `localstack` CLI via `pip`. If you're downloading the `localstack` CLI as a [binary release](https://docs.localstack.cloud/getting-started/installation/#localstack-cli), then please use the proxy configuration UI described below.
+
 ### Resource-specific proxying
 
 As an alternative to forwarding _all_ requests for a particular service, you can also proxy only requests for _specific_ resources to AWS.
@@ -70,6 +72,14 @@ make_bucket: test123
 
 A more comprehensive sample, involving local Lambda functions combined with remote SQS queues and S3 buckets, can be found in the `example` folder of this repo.
 
+### Proxy Configuration UI
+
+Once the extension is installed, it will expose a small configuration endpoint in your LocalStack container under the following endpoint: http://localhost:4566/_localstack/aws-replicator/index.html . 
+
+Please use this Web UI to define the proxy configuration (in YAML syntax), as well as the AWS credentials (AWS access key ID, secret access key, and optionally session token).
+
+![configuration settings](etc/proxy-settings.png)
+
 ## Resource Replicator CLI
 
 The figure below illustrates how the extension can be used to replicate the state, e.g., an SQS queue and the messages contained in it, from AWS into your LocalStack instance.
@@ -103,6 +113,11 @@ To install the extension itself (server component running inside LocalStack), us
 localstack extensions install "git+https://github.com/localstack/localstack-extensions/#egg=localstack-extension-aws-replicator&subdirectory=aws-replicator"
 ```
 
+## Change Log
+
+* `0.1.1`: Add simple configuration Web UI
+* `0.1.0`: Initial version of extension
+
 ## License
 
 This extension is published under the Apache License, Version 2.0.

aws-replicator/aws_replicator/client/auth_proxy.py

@@ -12,6 +12,7 @@
 import requests
 from botocore.awsrequest import AWSPreparedRequest
 from botocore.model import OperationModel
+from localstack import config
 from localstack import config as localstack_config
 from localstack.aws.api import HttpRequest
 from localstack.aws.protocol.parser import create_parser
@@ -35,10 +36,17 @@
 from aws_replicator.shared.models import AddProxyRequest, ProxyConfig
 
 LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.INFO)
+if config.DEBUG:
+    LOG.setLevel(logging.DEBUG)
 
 # TODO make configurable
 CLI_PIP_PACKAGE = "git+https://github.com/localstack/localstack-extensions/@main#egg=localstack-extension-aws-replicator&subdirectory=aws-replicator"
 
+CONTAINER_NAME_PREFIX = "ls-aws-proxy-"
+CONTAINER_CONFIG_FILE = "/tmp/ls.aws.proxy.yml"
+CONTAINER_LOG_FILE = "/tmp/ls-aws-proxy.log"
+
 
 class AuthProxyAWS(Server):
     def __init__(self, config: ProxyConfig, port: int = None):
@@ -250,7 +258,9 @@ def start_aws_auth_proxy(config: ProxyConfig, port: int = None) -> AuthProxyAWS:
     return proxy
 
 
-def start_aws_auth_proxy_in_container(config: ProxyConfig):
+def start_aws_auth_proxy_in_container(
+    config: ProxyConfig, env_vars: dict = None, port: int = None, quiet: bool = False
+):
     """
     Run the auth proxy in a separate local container. This can help in cases where users
     are running into version/dependency issues on their host machines.
@@ -267,23 +277,23 @@ def start_aws_auth_proxy_in_container(config: ProxyConfig):
 
     # determine port mapping
     localstack_config.PORTS_CHECK_DOCKER_IMAGE = DOCKER_IMAGE_NAME_PRO
-    port = reserve_available_container_port()
+    port = port or reserve_available_container_port()
     ports = PortMappings()
     ports.add(port, port)
 
     # create container
-    container_name = f"ls-aws-proxy-{short_uid()}"
+    container_name = f"{CONTAINER_NAME_PREFIX}{short_uid()}"
     image_name = DOCKER_IMAGE_NAME_PRO
     DOCKER_CLIENT.create_container(
         image_name,
         name=container_name,
         entrypoint="",
-        command=["bash", "-c", "while true; do sleep 1; done"],
+        command=["bash", "-c", f"touch {CONTAINER_LOG_FILE}; tail -f {CONTAINER_LOG_FILE}"],
         ports=ports,
     )
 
     # start container in detached mode
-    DOCKER_CLIENT.start_container(container_name)
+    DOCKER_CLIENT.start_container(container_name, attach=False)
 
     # install extension CLI package
     venv_activate = ". .venv/bin/activate"
@@ -297,9 +307,8 @@ def start_aws_auth_proxy_in_container(config: ProxyConfig):
     # create config file in container
     config_file_host = new_tmp_file()
     save_file(config_file_host, json.dumps(config))
-    config_file_cnt = "/tmp/ls.aws.proxy.yml"
     DOCKER_CLIENT.copy_into_container(
-        container_name, config_file_host, container_path=config_file_cnt
+        container_name, config_file_host, container_path=CONTAINER_CONFIG_FILE
     )
 
     # prepare environment variables
@@ -311,29 +320,42 @@ def start_aws_auth_proxy_in_container(config: ProxyConfig):
         "AWS_DEFAULT_REGION",
         ENV_LOCALSTACK_API_KEY,
     ]
-    env_vars = select_attributes(dict(os.environ), env_var_names)
+    env_vars = env_vars or os.environ
+    env_vars = select_attributes(dict(env_vars), env_var_names)
     env_vars["LOCALSTACK_HOSTNAME"] = "host.docker.internal"
 
     try:
-        env_vars_list = []
-        for key, value in env_vars.items():
-            env_vars_list += ["-e", f"{key}={value}"]
-        # note: using docker command directly, as our Docker client doesn't fully support log piping yet
-        command = [
-            "docker",
-            "exec",
-            "-it",
-            *env_vars_list,
-            container_name,
-            "bash",
-            "-c",
-            f"{venv_activate}; localstack aws proxy -c {config_file_cnt} -p {port}",
-        ]
         print("Proxy container is ready.")
-        subprocess.run(command, stdout=sys.stdout, stderr=sys.stderr)
+        command = f"{venv_activate}; localstack aws proxy -c {CONTAINER_CONFIG_FILE} -p {port} > {CONTAINER_LOG_FILE} 2>&1"
+        if quiet:
+            DOCKER_CLIENT.exec_in_container(
+                container_name, command=["bash", "-c", command], env_vars=env_vars, interactive=True
+            )
+        else:
+            env_vars_list = []
+            for key, value in env_vars.items():
+                env_vars_list += ["-e", f"{key}={value}"]
+            # note: using docker command directly, as our Docker client doesn't fully support log piping yet
+            command = [
+                "docker",
+                "exec",
+                "-it",
+                *env_vars_list,
+                container_name,
+                "bash",
+                "-c",
+                command,
+            ]
+            subprocess.run(command, stdout=sys.stdout, stderr=sys.stderr)
     except KeyboardInterrupt:
         pass
     except Exception as e:
-        print("Error:", e)
+        LOG.info("Error: %s", e)
+        if isinstance(e, subprocess.CalledProcessError):
+            LOG.info("Error in called process - output: %s\n%s", e.stdout, e.stderr)
     finally:
-        DOCKER_CLIENT.remove_container(container_name, force=True)
+        try:
+            DOCKER_CLIENT.remove_container(container_name, force=True)
+        except Exception as e:
+            if "already in progress" not in str(e):
+                raise

aws-replicator/aws_replicator/client/replicate.py

@@ -6,7 +6,6 @@
 from typing import Dict, List
 
 import boto3
-from localstack.services.cloudformation.engine import template_deployer
 from localstack.utils.collections import select_attributes
 from localstack.utils.files import load_file, save_file
 from localstack.utils.json import extract_jsonpath
@@ -167,6 +166,8 @@ def get_resources_custom(self, resource_type: str) -> List[Dict]:
         if not details:
             return []
 
+        from localstack.services.cloudformation.engine import template_deployer
+
         service_name = template_deployer.get_service_name({"Type": resource_type})
         from_client = boto3.client(service_name)
 
@@ -222,8 +223,12 @@ def create(self, resource: Dict):
         if model_instance:
             model_instance.add_extended_state_external()
 
+    def create_all(self):
+        # request creation
+        post_request_to_instance()
+
 
-def replicate_state(
+def replicate_state_with_scraper_on_host(
     scraper: AwsAccountScraper, creator: ResourceReplicator, services: List[str] = None
 ):
     """Replicate the state from a source AWS account into a target account (or LocalStack)"""
@@ -243,7 +248,18 @@ def replicate_state(
             creator.create(resource)
 
 
+def replicate_state_with_scraper_in_container(
+    creator: ResourceReplicator, services: List[str] = None
+):
+    """Replicate the state from a source AWS account into a target account (or LocalStack)"""
+    creator.create_all()
+
+
 def replicate_state_into_local(services: List[str]):
-    scraper = AwsAccountScraper(boto3.Session())
     creator = ResourceReplicatorClient()
-    return replicate_state(scraper, creator, services=services)
+
+    # deprecated
+    # scraper = AwsAccountScraper(boto3.Session())
+    # return replicate_state_with_scraper_on_host(scraper, creator, services=services)
+
+    return replicate_state_with_scraper_in_container(creator, services=services)

aws-replicator/aws_replicator/client/utils.py

@@ -9,9 +9,9 @@
 from aws_replicator.shared.models import ReplicateStateRequest
 
 
-def post_request_to_instance(request: ReplicateStateRequest):
+def post_request_to_instance(request: ReplicateStateRequest = None):
     url = f"{get_edge_url()}{HANDLER_PATH_REPLICATE}"
-    response = requests.post(url, json=request)
+    response = requests.post(url, json=request or {})
     if not response.ok:
         raise Exception(f"Invocation failed (code {response.status_code}): {response.content}")
     return response

aws-replicator/aws_replicator/server/aws_request_forwarder.py

@@ -6,7 +6,12 @@
 import requests
 from localstack.aws.api import RequestContext
 from localstack.aws.chain import Handler, HandlerChain
-from localstack.constants import APPLICATION_JSON, LOCALHOST, LOCALHOST_HOSTNAME
+from localstack.constants import (
+    APPLICATION_JSON,
+    LOCALHOST,
+    LOCALHOST_HOSTNAME,
+    TEST_AWS_ACCESS_KEY_ID,
+)
 from localstack.http import Response
 from localstack.utils.aws import arns
 from localstack.utils.aws.arns import sqs_queue_arn
@@ -93,13 +98,19 @@ def _request_matches_resource(
     ) -> bool:
         if context.service.service_name == "s3":
             bucket_name = context.service_request.get("Bucket") or ""
-            s3_bucket_arn = arns.s3_bucket_arn(bucket_name, account_id=context.account_id)
+            s3_bucket_arn = arns.s3_bucket_arn(bucket_name)
             return bool(re.match(resource_name_pattern, s3_bucket_arn))
         if context.service.service_name == "sqs":
             queue_name = context.service_request.get("QueueName") or ""
             queue_url = context.service_request.get("QueueUrl") or ""
             queue_name = queue_name or queue_url.split("/")[-1]
-            candidates = (queue_name, queue_url, sqs_queue_arn(queue_name))
+            candidates = (
+                queue_name,
+                queue_url,
+                sqs_queue_arn(
+                    queue_name, account_id=context.account_id, region_name=context.region
+                ),
+            )
             for candidate in candidates:
                 if re.match(resource_name_pattern, candidate):
                     return True
@@ -176,7 +187,9 @@ def _extract_region_from_domain(self, context: RequestContext):
         for part in parts:
             if part in valid_regions:
                 context.request.headers["Authorization"] = mock_aws_request_headers(
-                    context.service.service_name, region_name=part
+                    context.service.service_name,
+                    region_name=part,
+                    aws_access_key_id=TEST_AWS_ACCESS_KEY_ID,
                 )
                 return
 

aws-replicator/aws_replicator/server/extension.py

@@ -11,13 +11,10 @@ class AwsReplicatorExtension(Extension):
     name = "aws-replicator"
 
     def update_gateway_routes(self, router: http.Router[http.RouteHandler]):
-        from aws_replicator.config import HANDLER_PATH_PROXIES, HANDLER_PATH_REPLICATE
         from aws_replicator.server.request_handler import RequestHandler
 
         LOG.info("AWS resource replicator: adding routes to activate extension")
-        endpoint = RequestHandler()
-        get_internal_apis().add(HANDLER_PATH_REPLICATE, endpoint, methods=["POST"])
-        get_internal_apis().add(HANDLER_PATH_PROXIES, endpoint, methods=["POST"])
+        get_internal_apis().add(RequestHandler())
 
     def update_request_handlers(self, handlers: CompositeHandler):
         from aws_replicator.server.aws_request_forwarder import AwsProxyHandler