diff --git a/aws-replicator/Makefile b/aws-replicator/Makefile index 6b60fd2..4544cc2 100644 --- a/aws-replicator/Makefile +++ b/aws-replicator/Makefile @@ -33,7 +33,7 @@ install: venv $(VENV_RUN); $(PIP_CMD) install -e ".[test]" test: venv - $(VENV_RUN); python -m pytest $(TEST_PATH) + $(VENV_RUN); python -m pytest $(PYTEST_ARGS) $(TEST_PATH) dist: venv $(VENV_RUN); python setup.py sdist bdist_wheel diff --git a/aws-replicator/README.md b/aws-replicator/README.md index d1661bc..839ae67 100644 --- a/aws-replicator/README.md +++ b/aws-replicator/README.md @@ -126,6 +126,7 @@ If you wish to access the deprecated instructions, they can be found [here](http ## Change Log +* `0.1.20`: Fix logic for proxying S3 requests with `*.s3.amazonaws.com` host header * `0.1.19`: Print human-readable message for invalid regexes in resource configs; fix logic for proxying S3 requests with host-based addressing * `0.1.18`: Update environment check to use SDK Docker client and enable starting the proxy from within Docker (e.g., from the LS main container as part of an init script) * `0.1.17`: Add basic support for ARN-based pattern-matching for `secretsmanager` resources diff --git a/aws-replicator/aws_replicator/client/auth_proxy.py b/aws-replicator/aws_replicator/client/auth_proxy.py index 895b6e1..ecd8afb 100644 --- a/aws-replicator/aws_replicator/client/auth_proxy.py +++ b/aws-replicator/aws_replicator/client/auth_proxy.py @@ -275,9 +275,10 @@ def _fix_headers(self, request: Request, service_name: str): def _fix_host_and_path(self, request: Request, service_name: str): if service_name == "s3": # fix the path and prepend the bucket name, to avoid bucket addressing issues + regex_base_domain = rf"((amazonaws\.com)|({LOCALHOST_HOSTNAME}))" host = request.headers.pop(HEADER_HOST_ORIGINAL, None) host = host or request.headers.get("Host") or "" - match = re.match(rf"(.+)\.s3\.{LOCALHOST_HOSTNAME}", host) + match = re.match(rf"(.+)\.s3\.{regex_base_domain}", host) if match: # prepend the bucket name (extracted from the host) to the path of the request (path-based addressing) request.path = f"/{match.group(1)}{request.path}" diff --git a/aws-replicator/setup.cfg b/aws-replicator/setup.cfg index d7a0546..c371e87 100644 --- a/aws-replicator/setup.cfg +++ b/aws-replicator/setup.cfg @@ -1,8 +1,8 @@ [metadata] name = localstack-extension-aws-replicator -version = 0.1.19 -summary = LocalStack Extension: AWS replicator -description = Replicate AWS resources into your LocalStack instance +version = 0.1.20 +summary = LocalStack AWS Proxy Extension +description = Proxy AWS resources into your LocalStack instance long_description = file: README.md long_description_content_type = text/markdown; charset=UTF-8 url = https://github.com/localstack/localstack-extensions/tree/main/aws-replicator diff --git a/aws-replicator/tests/test_proxy_requests.py b/aws-replicator/tests/test_proxy_requests.py index 8fdfe34..148fc91 100644 --- a/aws-replicator/tests/test_proxy_requests.py +++ b/aws-replicator/tests/test_proxy_requests.py @@ -1,6 +1,8 @@ # Note: these tests depend on the extension being installed and actual AWS credentials being configured, such # that the proxy can be started within the tests. They are designed to be mostly run in CI at this point. import gzip +import re +from urllib.parse import urlparse import boto3 import pytest @@ -41,20 +43,33 @@ def _start(config: dict = None): @pytest.mark.parametrize("metadata_gzip", [True, False]) -@pytest.mark.parametrize("host_addressing", [True, False]) -def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip, host_addressing): +@pytest.mark.parametrize("target_endpoint", ["local_domain", "aws_domain", "default"]) +def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip, target_endpoint): # start proxy config = ProxyConfig(services={"s3": {"resources": ".*"}}, bind_host=PROXY_BIND_HOST) start_aws_proxy(config) # create clients - if host_addressing: + if target_endpoint == "default": + s3_client = connect_to().s3 + else: s3_client = connect_to( endpoint_url="http://s3.localhost.localstack.cloud:4566", config=Config(s3={"addressing_style": "virtual"}), ).s3 - else: - s3_client = connect_to().s3 + + if target_endpoint == "aws_domain": + + def _add_header(request, **kwargs): + # instrument boto3 client to add custom `Host` header, mimicking a `*.s3.amazonaws.com` request + url = urlparse(request.url) + match = re.match(r"(.+)\.s3\.localhost\.localstack\.cloud", url.netloc) + if match: + request.headers.add_header("host", f"{match.group(1)}.s3.amazonaws.com") + + s3_client.meta.events.register_first("before-sign.*.*", _add_header) + + # define S3 client pointing to real AWS s3_client_aws = boto3.client("s3") # list buckets to assert that proxy is up and running