From 9ef83b6b59ecaad9566e4237fca1ca2405f482be Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 23 Nov 2024 11:39:48 -0500 Subject: [PATCH] pin wrangler version to fix hanging miniflare invocations (#86) --- .../aws_replicator/client/auth_proxy.py | 2 +- aws-replicator/tests/test_proxy_requests.py | 2 +- miniflare/Makefile | 4 ++-- miniflare/README.md | 1 + miniflare/miniflare/cloudflare_api.py | 12 ++++++++++-- miniflare/miniflare/extension.py | 17 ++++++++++------- miniflare/setup.cfg | 2 +- 7 files changed, 26 insertions(+), 14 deletions(-) diff --git a/aws-replicator/aws_replicator/client/auth_proxy.py b/aws-replicator/aws_replicator/client/auth_proxy.py index ecd8afb..f3ecd48 100644 --- a/aws-replicator/aws_replicator/client/auth_proxy.py +++ b/aws-replicator/aws_replicator/client/auth_proxy.py @@ -278,7 +278,7 @@ def _fix_host_and_path(self, request: Request, service_name: str): regex_base_domain = rf"((amazonaws\.com)|({LOCALHOST_HOSTNAME}))" host = request.headers.pop(HEADER_HOST_ORIGINAL, None) host = host or request.headers.get("Host") or "" - match = re.match(rf"(.+)\.s3\.{regex_base_domain}", host) + match = re.match(rf"(.+)\.s3\..*{regex_base_domain}", host) if match: # prepend the bucket name (extracted from the host) to the path of the request (path-based addressing) request.path = f"/{match.group(1)}{request.path}" diff --git a/aws-replicator/tests/test_proxy_requests.py b/aws-replicator/tests/test_proxy_requests.py index 148fc91..c1518f9 100644 --- a/aws-replicator/tests/test_proxy_requests.py +++ b/aws-replicator/tests/test_proxy_requests.py @@ -65,7 +65,7 @@ def _add_header(request, **kwargs): url = urlparse(request.url) match = re.match(r"(.+)\.s3\.localhost\.localstack\.cloud", url.netloc) if match: - request.headers.add_header("host", f"{match.group(1)}.s3.amazonaws.com") + request.headers.add_header("host", f"{match.group(1)}.s3.us-east-1.amazonaws.com") s3_client.meta.events.register_first("before-sign.*.*", _add_header) diff --git a/miniflare/Makefile b/miniflare/Makefile index 218eb31..f59dcbf 100644 --- a/miniflare/Makefile +++ b/miniflare/Makefile @@ -18,10 +18,10 @@ clean: rm -rf *.egg-info/ lint: ## Run code linter to check code style - $(VENV_RUN); python -m pflake8 --show-source --ignore=E501 + $(VENV_RUN); python -m pflake8 --show-source --ignore=E501 --exclude .venv,build format: ## Run black and isort code formatter - $(VENV_RUN); python -m isort .; python -m black . + $(VENV_RUN); python -m isort .; python -m black miniflare install: venv $(VENV_RUN); python -m pip install -e .[dev] diff --git a/miniflare/README.md b/miniflare/README.md index 8c1aba8..fcb73e9 100644 --- a/miniflare/README.md +++ b/miniflare/README.md @@ -34,6 +34,7 @@ Hello World! ## Change Log +* `0.1.2`: Pin wrangler version to fix hanging miniflare invocations; fix encoding headers for invocation responses * `0.1.1`: Adapt for LocalStack v3.0 * `0.1.0`: Upgrade to Miniflare 3.0 * `0.0.1`: Initial version. diff --git a/miniflare/miniflare/cloudflare_api.py b/miniflare/miniflare/cloudflare_api.py index e822233..4ec467c 100644 --- a/miniflare/miniflare/cloudflare_api.py +++ b/miniflare/miniflare/cloudflare_api.py @@ -42,13 +42,19 @@ def handle_invocation(request: Request, path: str, script_name: str, port: str): port = SCRIPT_SERVERS[script_name].port response = requests.request( method=request.method, - url=f"http://localhost:{port}{request.path}", + url=f"http://localhost:{port}/{path}", data=request.get_data(), ) result = Response() result.status_code = response.status_code result.set_data(response.content) - result.headers.update(dict(response.headers)) + headers = dict(response.headers) + if headers.get('Transfer-Encoding') == 'chunked': + headers.pop('Transfer-Encoding') + if headers.get('Content-Encoding') == 'gzip': + headers.pop('Content-Encoding') + LOG.debug("Miniflare invocation response headers/body: %s / %s", headers, response.content) + result.headers.update(headers) return result @@ -154,9 +160,11 @@ def handle_services(request: Request, account_id: str, service_name: str) -> dic } ) + def handle_standard(request: Request, account_id: str) -> dict: return _wrap({}) + def handle_subdomain(request: Request, account_id: str) -> dict: return _wrap({}) diff --git a/miniflare/miniflare/extension.py b/miniflare/miniflare/extension.py index a123e70..ca2f350 100644 --- a/miniflare/miniflare/extension.py +++ b/miniflare/miniflare/extension.py @@ -26,8 +26,9 @@ LOG = logging.getLogger(__name__) -# identifier for default version installed by package installer -DEFAULT_VERSION = "latest" +# Identifier for default version of `wrangler` installed by package installer. +# Note: Currently pinned to 3.1.0, as newer versions make the invocations hang in the LS container +WRANGLER_VERSION = "3.1.0" class MiniflareExtension(Extension): @@ -36,6 +37,10 @@ class MiniflareExtension(Extension): def update_gateway_routes(self, router: http.Router[http.RouteHandler]): from miniflare.config import HANDLER_PATH_MINIFLARE + logging.getLogger("miniflare").setLevel( + logging.DEBUG if config.DEBUG else logging.INFO + ) + LOG.info("miniflare: adding routes to activate extension") all_methods = ["GET", "POST", "PUT", "DELETE"] @@ -83,7 +88,7 @@ def __init__(self, script: WorkerScript, port: int): super().__init__(port) def do_run(self): - root_dir = os.path.join(config.dirs.var_libs, "miniflare", DEFAULT_VERSION) + root_dir = os.path.join(config.dirs.var_libs, "miniflare", WRANGLER_VERSION) wrangler_bin = os.path.join(root_dir, "node_modules", ".bin", "wrangler") # add global aliases, and variable bindings @@ -102,7 +107,6 @@ def do_run(self): cmd = [ wrangler_bin, "dev", - "--experimental-local", "--port", str(self.port), script_path_final, @@ -115,7 +119,7 @@ def do_run(self): class MiniflareInstaller(ExecutableInstaller): def __init__(self): - super().__init__("miniflare", version=DEFAULT_VERSION) + super().__init__("miniflare", version=WRANGLER_VERSION) def _get_install_marker_path(self, install_dir: str) -> str: # force re-install on every start (requires npm package + system libs like libc++) @@ -134,5 +138,4 @@ def _install(self, target: InstallTarget) -> None: run(["apt", "install", "-y", "libc++-dev"]) # install npm package - run(["npm", "install", "--prefix", target_dir, "wrangler"]) - run(["npm", "install", "--prefix", target_dir, "@miniflare/tre"]) + run(["npm", "install", "--prefix", target_dir, f"wrangler@{WRANGLER_VERSION}"]) diff --git a/miniflare/setup.cfg b/miniflare/setup.cfg index d668b3e..8a16a1f 100644 --- a/miniflare/setup.cfg +++ b/miniflare/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = localstack-extension-miniflare -version = 0.1.2 +version = 0.1.3 summary = LocalStack Extension: Miniflare description = This extension makes Miniflare (dev environment for Cloudflare workers) available directly in LocalStack long_description = file: README.md