From e8c56f083a6bf3c28683575e7b2ff12913f5fc5e Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Wed, 20 Sep 2023 10:14:49 -0500 Subject: [PATCH 01/43] :package: Updates dev reqs --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 74fd354..a1b1b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,11 +41,11 @@ requires-python = ">=3.8" [project.optional-dependencies] dev = [ - "black", "coverage[toml]", "django-stubs", "django-stubs-ext", "hatch", + "model_bakery", "mypy", "nox", "pytest", From 9bac050f40ea3a4c27ce58e1ca94c4cdd7692443 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Wed, 20 Sep 2023 10:36:33 -0500 Subject: [PATCH 02/43] :gear: Updates test settings --- tests/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/settings.py b/tests/settings.py index 308d87d..fa28e08 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -16,13 +16,14 @@ } DATABASE_ROUTERS = [ - "email_relay.db.EmailRelayDatabaseRouter", + "email_relay.db.EmailDatabaseRouter", ] EMAIL_BACKEND = "email_relay.backend.DatabaseEmailBackend" INSTALLED_APPS = [ + "django.contrib.contenttypes", "email_relay", ] From b8eb114faea5d18198323abb222e22a5f6efaf4f Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Wed, 20 Sep 2023 10:46:11 -0500 Subject: [PATCH 03/43] :tractor: Refactor test settings --- pyproject.toml | 4 ---- tests/conftest.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ tests/settings.py | 32 -------------------------------- 3 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 tests/conftest.py delete mode 100644 tests/settings.py diff --git a/pyproject.toml b/pyproject.toml index a1b1b64..86f53da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,9 +119,6 @@ omit = [ ] source = ["src/email_relay"] -[tool.django-stubs] -django_settings_module = "tests.settings" - [tool.mypy] check_untyped_defs = true files = [ @@ -149,7 +146,6 @@ module = [] ignore_missing_model_attributes = true [tool.pytest.ini_options] -DJANGO_SETTINGS_MODULE = "tests.settings" django_find_project = false pythonpath = ". src" addopts = "--create-db -n auto --dist loadfile" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bc85346 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import logging + +import pytest +from django.conf import settings +from django.test.utils import override_settings + +from email_relay.conf import EMAIL_RELAY_DATABASE_ALIAS + +pytest_plugins = [] + + +# Settings fixtures to bootstrap our tests +def pytest_configure(config): + logging.disable(logging.CRITICAL) + + settings.configure( + ALLOWED_HOSTS = ["*"], + CACHES={ + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + } + }, + DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + }, + EMAIL_RELAY_DATABASE_ALIAS: { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + }, + }, + DATABASE_ROUTERS = [ + "email_relay.db.EmailDatabaseRouter", + ], + EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend", + INSTALLED_APPS = [ + "django.contrib.contenttypes", + "email_relay", + ], + LOGGING_CONFIG = None, + PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"], + SECRET_KEY = "NOTASECRET", + USE_TZ = True, + ) diff --git a/tests/settings.py b/tests/settings.py deleted file mode 100644 index fa28e08..0000000 --- a/tests/settings.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -from email_relay.conf import EMAIL_RELAY_DATABASE_ALIAS - -ALLOWED_HOSTS = ["*"] - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": ":memory:", - }, - EMAIL_RELAY_DATABASE_ALIAS: { - "ENGINE": "django.db.backends.sqlite3", - "NAME": ":memory:", - }, -} - -DATABASE_ROUTERS = [ - "email_relay.db.EmailDatabaseRouter", -] - - -EMAIL_BACKEND = "email_relay.backend.DatabaseEmailBackend" - -INSTALLED_APPS = [ - "django.contrib.contenttypes", - "email_relay", -] - -SECRET_KEY = "NOTASECRET" - -USE_TZ = True From 07dc661d04cdbab79ec99e53dc155c0fb75bb92d Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Wed, 20 Sep 2023 10:46:43 -0500 Subject: [PATCH 04/43] :green_heart: Adds a simple model test This is what we were working towards and why we made the other changes. --- tests/test_models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/test_models.py diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..d1f1bb0 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import pytest +from model_bakery import baker + +from email_relay.models import Message + + +@pytest.mark.django_db(databases=["default", "email_relay_db"]) +def test_message(): + baker.make("email_relay.Message") + assert Message.objects.all().count() == 1 From d1063c069d520d85ac815d87c6de3d07597fa584 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:57:52 +0000 Subject: [PATCH 05/43] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/conftest.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bc85346..10f2245 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,7 @@ import logging -import pytest from django.conf import settings -from django.test.utils import override_settings from email_relay.conf import EMAIL_RELAY_DATABASE_ALIAS @@ -16,13 +14,13 @@ def pytest_configure(config): logging.disable(logging.CRITICAL) settings.configure( - ALLOWED_HOSTS = ["*"], + ALLOWED_HOSTS=["*"], CACHES={ "default": { "BACKEND": "django.core.cache.backends.dummy.DummyCache", } }, - DATABASES = { + DATABASES={ "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:", @@ -32,16 +30,16 @@ def pytest_configure(config): "NAME": ":memory:", }, }, - DATABASE_ROUTERS = [ + DATABASE_ROUTERS=[ "email_relay.db.EmailDatabaseRouter", ], - EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend", - INSTALLED_APPS = [ + EMAIL_BACKEND="django.core.mail.backends.locmem.EmailBackend", + INSTALLED_APPS=[ "django.contrib.contenttypes", "email_relay", ], - LOGGING_CONFIG = None, - PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"], - SECRET_KEY = "NOTASECRET", - USE_TZ = True, + LOGGING_CONFIG=None, + PASSWORD_HASHERS=["django.contrib.auth.hashers.MD5PasswordHasher"], + SECRET_KEY="NOTASECRET", + USE_TZ=True, ) From 7383915bd98f5a0ee8267fd72e077be4ebc31eb2 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 12:18:43 -0500 Subject: [PATCH 06/43] :green_heart: Adds router tests --- tests/test_router.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tests/test_router.py diff --git a/tests/test_router.py b/tests/test_router.py new file mode 100644 index 0000000..146a52c --- /dev/null +++ b/tests/test_router.py @@ -0,0 +1,38 @@ +import pytest +from email_relay.conf import app_settings +from email_relay.db import EmailDatabaseRouter + + +# Mock model with app_label "email_relay" +class MockModel: + class _meta: + app_label = "email_relay" + + +# Mock model with app_label "some_other_app" +class MockModelOther: + class _meta: + app_label = "some_other_app" + + +@pytest.fixture +def router(): + return EmailDatabaseRouter() + + +def test_db_for_read(router): + assert router.db_for_read(MockModel) == app_settings.DATABASE_ALIAS + assert router.db_for_read(MockModelOther) == "default" + + +def test_db_for_write(router): + assert router.db_for_write(MockModel) == app_settings.DATABASE_ALIAS + assert router.db_for_write(MockModelOther) == "default" + + +def test_allow_relation(router): + assert router.allow_relation(None, None) + + +def test_allow_migrate(router): + assert router.allow_migrate("some_db", "some_app_label") From 62391d1c275e34aa0007c7615f90385fe5fdd700 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 17:19:07 +0000 Subject: [PATCH 07/43] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_router.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_router.py b/tests/test_router.py index 146a52c..3ad6eb9 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import pytest + from email_relay.conf import app_settings from email_relay.db import EmailDatabaseRouter From f197c7334e4820cc65a6e878ca9e8c768691d671 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 13:04:52 -0500 Subject: [PATCH 08/43] :gear: :hammer: mypy fixes? --- pyproject.toml | 18 ++++++++---------- src/email_relay/backend.py | 2 +- tests/conftest.py | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86f53da..61551f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,8 +53,8 @@ dev = [ "pytest-randomly", "pytest-reverse", "pytest-xdist", - "ruff", ] +lint = ["pre-commit"] psycopg = ["psycopg[binary]"] psycopg2 = ["psycopg2-binary"] @@ -119,11 +119,14 @@ omit = [ ] source = ["src/email_relay"] +[tool.django-stubs] +django_settings_module = "tests.settings" +strict_settings = false + [tool.mypy] +mypy_path = "src/" +namespace_packages = false check_untyped_defs = true -files = [ - "src.email_relay", -] no_implicit_optional = true plugins = [ "mypy_django_plugin.main", @@ -134,13 +137,8 @@ warn_unused_ignores = true [[tool.mypy.overrides]] ignore_errors = true -module = [ - "src.email_relay.*.migrations.*", -] - -[[tool.mypy.overrides]] ignore_missing_imports = true -module = [] +module = "tests.*" [tool.mypy_django_plugin] ignore_missing_model_attributes = true diff --git a/src/email_relay/backend.py b/src/email_relay/backend.py index 8848c06..d17efce 100644 --- a/src/email_relay/backend.py +++ b/src/email_relay/backend.py @@ -12,7 +12,7 @@ class RelayDatabaseEmailBackend(BaseEmailBackend): def send_messages(self, email_messages: Sequence[EmailMessage]) -> int: messages = Message.objects.bulk_create( - [Message(email=email) for email in email_messages], # type: ignore[misc] + [Message(email=email) for email in email_messages], app_settings.MESSAGES_BATCH_SIZE, ) return len(messages) diff --git a/tests/conftest.py b/tests/conftest.py index 10f2245..651532a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,7 @@ from email_relay.conf import EMAIL_RELAY_DATABASE_ALIAS -pytest_plugins = [] +pytest_plugins = [] # type: ignore # Settings fixtures to bootstrap our tests From 9ecbbd9ff20bd9413dc43b721e7305523d4de317 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 13:16:38 -0500 Subject: [PATCH 09/43] :pencil: commits blank file to make mypy happy --- tests/settings.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/settings.py diff --git a/tests/settings.py b/tests/settings.py new file mode 100644 index 0000000..e69de29 From 6140dde29b2df886300ac2cec8e9cdf17ce09fd0 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 13:19:29 -0500 Subject: [PATCH 10/43] :tractor: Adds/updates lint+mypy tasks --- Justfile | 5 ++++- noxfile.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 866b53e..c817ced 100644 --- a/Justfile +++ b/Justfile @@ -235,4 +235,7 @@ envsync: ################## lint: - pre-commit run --all-files + python -m nox --reuse-existing-virtualenvs --session "lint" + +mypy: + python -m nox --reuse-existing-virtualenvs --session "mypy" diff --git a/noxfile.py b/noxfile.py index 74e09fa..35f752e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -51,3 +51,15 @@ def tests(session, django): session.install(f"django=={django}") session.run("pytest", "-n", "auto", "--dist", "loadfile") + + +@nox.session +def lint(session): + session.install(".[lint]") + session.run("pre-commit", "run", "--all-files") + + +@nox.session +def mypy(session): + session.install(".[dev]") + session.run("mypy") From 426f7cd2a3370cb680e2fe1d008f32e0666f9eea Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 14:10:57 -0500 Subject: [PATCH 11/43] :gear: Fixes coverage ignore migrations --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 61551f6..722f0a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ version_pattern = "YYYY.INC1" [tool.coverage.run] omit = [ - "src/email_relay/*/migrations/*", + "src/email_relay/migrations/*", "tests/*", "manage.py", "service.py" From b872c1a1ee76bd6662bf6d482f13a228e99e727a Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 14:11:19 -0500 Subject: [PATCH 12/43] :gear: Sets coverage to 33% (we will increase this) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d5fc9d1..4c9754a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,4 +91,4 @@ jobs: coverage run -m pytest python -m coverage html --skip-covered --skip-empty python -m coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY - python -m coverage report --fail-under=100 + python -m coverage report --fail-under=33 From 7ff31c560c7227f85d3a6ab3ed444b8581dc1090 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 14:30:54 -0500 Subject: [PATCH 13/43] :gear: :arrow_down: Drop coverage --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c9754a..306ea83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,4 +91,4 @@ jobs: coverage run -m pytest python -m coverage html --skip-covered --skip-empty python -m coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY - python -m coverage report --fail-under=33 + python -m coverage report --fail-under=30 From 7187cecb50f056d71a8698f83c91aa11b2d07148 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 15:39:24 -0500 Subject: [PATCH 14/43] :gear: Configures pytest-coverage support --- Justfile | 7 ++++--- pyproject.toml | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index c817ced..bf5af13 100644 --- a/Justfile +++ b/Justfile @@ -48,9 +48,10 @@ test: python -m nox --reuse-existing-virtualenvs coverage: - rm -rf htmlcov - python -m coverage run -m pytest - python -m coverage html --skip-covered --skip-empty + rm -rf .coverage htmlcov + pytest -vv + python -m coverage html --skip-empty # --skip-covered + python -m coverage report --fail-under=100 types: python -m mypy . diff --git a/pyproject.toml b/pyproject.toml index 722f0a5..a1d9fad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dev = [ "mypy", "nox", "pytest", + "pytest-cov", "pytest-django", "pytest-randomly", "pytest-reverse", @@ -112,12 +113,16 @@ version_pattern = "YYYY.INC1" [tool.coverage.run] omit = [ + "manage.py", + "service.py", "src/email_relay/migrations/*", "tests/*", - "manage.py", - "service.py" ] -source = ["src/email_relay"] +source = ["email_relay"] + +[tool.coverage.paths] +source = ["src"] + [tool.django-stubs] django_settings_module = "tests.settings" @@ -146,9 +151,10 @@ ignore_missing_model_attributes = true [tool.pytest.ini_options] django_find_project = false pythonpath = ". src" -addopts = "--create-db -n auto --dist loadfile" +addopts = "--create-db --cov=email_relay -n auto --dist loadfile" norecursedirs = ".* bin build dist *.egg htmlcov logs node_modules templates venv" python_files = "tests.py test_*.py *_tests.py" +testpaths = ["tests"] [tool.ruff] ignore = ["E501", "E741"] # temporary From 1dcd1bd19607d18b4051659152da4449f23a57ac Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 15:45:06 -0500 Subject: [PATCH 15/43] :green_heart: Adds test_runrelay cmd --- tests/test_runrelay.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/test_runrelay.py diff --git a/tests/test_runrelay.py b/tests/test_runrelay.py new file mode 100644 index 0000000..7f0be98 --- /dev/null +++ b/tests/test_runrelay.py @@ -0,0 +1,13 @@ +import pytest +from django.core.management import call_command + + +def test_runrelay_help(): + # We'll capture the output of the command + with pytest.raises(SystemExit) as exec_info: + # call_command will execute our command as if we ran it from the command line + # the 'stdout' argument captures the command output + call_command("runrelay", "--help") + + # Asserting that the command exits with a successful exit code (0 for help command) + assert exec_info.value.code == 0 From 73a4e27b16009e9903880d297ef85cd9cce7d6c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 20:45:17 +0000 Subject: [PATCH 16/43] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_runrelay.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_runrelay.py b/tests/test_runrelay.py index 7f0be98..498a3f7 100644 --- a/tests/test_runrelay.py +++ b/tests/test_runrelay.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import pytest from django.core.management import call_command From f8b785a87bcd2cba34db259e19755d9915befaba Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 20 Sep 2023 15:55:26 -0500 Subject: [PATCH 17/43] add intro to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index aea8c6c..b6953fa 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ +`django-email-relay` enables Django projects without direct access to a preferred SMTP server to use that server for email dispatch. It consists of two parts: a Django app with a custom email backend that stores emails in a central database queue, and a relay service that reads from this queue to orchestrate email sending, available as either a standalone Docker image or a management command to be used within a Django project that does have access to the preferred SMTP server. + +Why opt for this setup? One reason is the potential for emails sent through an external Email Service Provider (ESP) to be marked as spam or filtered, a common issue when routing transactional emails from internal applications to internal users via an ESP. Moreover, it eliminates the necessity to open firewall ports or to need to utilize services like Tailscale for SMTP server access. Additionally, this approach decouples the emailing process from the main web application, enhancing both performance and reliability. + ## Requirements - Python 3.8, 3.9, 3.10, 3.11, or 3.12 From 3edfaf3d6900bf855dbe16b9ef9b6d317b97d0cb Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 20 Sep 2023 15:58:34 -0500 Subject: [PATCH 18/43] adjust formatting --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b6953fa..3a29cb8 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,15 @@ -`django-email-relay` enables Django projects without direct access to a preferred SMTP server to use that server for email dispatch. It consists of two parts: a Django app with a custom email backend that stores emails in a central database queue, and a relay service that reads from this queue to orchestrate email sending, available as either a standalone Docker image or a management command to be used within a Django project that does have access to the preferred SMTP server. +`django-email-relay` enables Django projects without direct access to a preferred SMTP server to use that server for email dispatch. -Why opt for this setup? One reason is the potential for emails sent through an external Email Service Provider (ESP) to be marked as spam or filtered, a common issue when routing transactional emails from internal applications to internal users via an ESP. Moreover, it eliminates the necessity to open firewall ports or to need to utilize services like Tailscale for SMTP server access. Additionally, this approach decouples the emailing process from the main web application, enhancing both performance and reliability. +It consists of two parts: a Django app with a custom email backend that stores emails in a central database queue, and a relay service that reads from this queue to orchestrate email sending, available as either a standalone Docker image or a management command to be used within a Django project that does have access to the preferred SMTP server. + +Why opt for this setup? + +- The potential for emails sent through an external Email Service Provider (ESP) to be marked as spam or filtered, a common issue when routing transactional emails from internal applications to internal users via an ESP. +- It eliminates the necessity to open firewall ports or to need to utilize services like Tailscale for SMTP server access. +- It decouples the emailing process from the main web application, much in the same way as using a task queue like Celery or Django-Q2 would. ## Requirements From f74db200c3efb921553c3fa5b0692f634ccf7431 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 20 Sep 2023 15:59:45 -0500 Subject: [PATCH 19/43] adjust wording --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a29cb8..095a1f1 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ It consists of two parts: a Django app with a custom email backend that stores e Why opt for this setup? - The potential for emails sent through an external Email Service Provider (ESP) to be marked as spam or filtered, a common issue when routing transactional emails from internal applications to internal users via an ESP. -- It eliminates the necessity to open firewall ports or to need to utilize services like Tailscale for SMTP server access. +- It eliminates the necessity to open firewall ports or the need to utilize services like Tailscale for SMTP server access. - It decouples the emailing process from the main web application, much in the same way as using a task queue like Celery or Django-Q2 would. ## Requirements From af7484a98012e26ebff8528f3e4ef5703fe8de8a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 21:01:01 +0000 Subject: [PATCH 20/43] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 095a1f1..745c4fc 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It consists of two parts: a Django app with a custom email backend that stores e Why opt for this setup? -- The potential for emails sent through an external Email Service Provider (ESP) to be marked as spam or filtered, a common issue when routing transactional emails from internal applications to internal users via an ESP. +- The potential for emails sent through an external Email Service Provider (ESP) to be marked as spam or filtered, a common issue when routing transactional emails from internal applications to internal users via an ESP. - It eliminates the necessity to open firewall ports or the need to utilize services like Tailscale for SMTP server access. - It decouples the emailing process from the main web application, much in the same way as using a task queue like Celery or Django-Q2 would. From f06d50b453020f18aa1ced07957c0f7cb4296452 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 15:38:54 -0500 Subject: [PATCH 21/43] add docker publish job to release workflow --- .github/workflows/release.yml | 55 ++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2eef435..603602b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,9 +4,20 @@ on: release: types: [created] workflow_dispatch: + inputs: + pypi: + description: "Publish to PyPI" + required: false + default: true + type: boolean + docker: + description: "Publish to GHCR" + required: false + default: true + type: boolean jobs: - publish: + pypi: runs-on: ubuntu-latest environment: release permissions: @@ -32,3 +43,45 @@ jobs: - if: ${{ github.event_name != 'workflow_dispatch' }} name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 + + docker: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/${{ github.repository }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=pep440,pattern={{version}} + type=pep440,pattern={{major}}.{{minor}} + type=pep440,pattern={{major}} + type=sha,prefix=sha- + type=raw,value=latest,enable={{is_default_branch}} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish Docker image + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max From fcfb62ba638286d16fa9300514cdefa4d70238cc Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 15:50:55 -0500 Subject: [PATCH 22/43] add if checks to release workflow --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 603602b..bb7dfb2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,7 @@ on: jobs: pypi: + if: ${{ github.event_name == 'release' || inputs.pypi }} runs-on: ubuntu-latest environment: release permissions: @@ -45,6 +46,7 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 docker: + if: ${{ github.event_name == 'release' || inputs.docker }} runs-on: ubuntu-latest permissions: contents: read From edb3e9b67ee8fe5026867bdc53b2176ff3f871de Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Thu, 21 Sep 2023 16:00:24 -0500 Subject: [PATCH 23/43] add correct DATABASES settings for PgBouncer (#7) --- service.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/service.py b/service.py index 81327db..1a07c6e 100644 --- a/service.py +++ b/service.py @@ -7,10 +7,16 @@ from django.conf import settings from django.core.management import call_command +DEBUG = os.getenv("EMAIL_RELAY_DEBUG", False) + SETTINGS = { - "DEBUG": os.getenv("EMAIL_RELAY_DEBUG", False), + "DEBUG": DEBUG, "DATABASES": { - "default": dj_database_url.parse(os.getenv("EMAIL_RELAY_DATABASE_URL", "")), + "default": dj_database_url.parse( + os.getenv("EMAIL_RELAY_DATABASE_URL", ""), + conn_max_age=600, # 10 minutes + conn_health_checks=True, + ), }, "LOGGING": { "version": 1, @@ -29,6 +35,8 @@ "email_relay", ], } +if not DEBUG: + SETTINGS["DATABASES"]["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True if __name__ == "__main__": settings.configure(**SETTINGS) From d5b01c8c36afe91ec68cded7a6a69d5747cf29ba Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Thu, 21 Sep 2023 21:05:01 -0500 Subject: [PATCH 24/43] add default email settings to service (#9) --- service.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/service.py b/service.py index 1a07c6e..8859f47 100644 --- a/service.py +++ b/service.py @@ -4,20 +4,45 @@ import dj_database_url import django +from django.conf import global_settings from django.conf import settings from django.core.management import call_command DEBUG = os.getenv("EMAIL_RELAY_DEBUG", False) SETTINGS = { - "DEBUG": DEBUG, "DATABASES": { "default": dj_database_url.parse( - os.getenv("EMAIL_RELAY_DATABASE_URL", ""), + os.getenv("DATABASE_URL", ""), conn_max_age=600, # 10 minutes conn_health_checks=True, ), }, + "DEBUG": DEBUG, + "DEFAULT_FROM_EMAIL": os.getenv( + "DEFAULT_FROM_EMAIL", global_settings.DEFAULT_FROM_EMAIL + ), + "EMAIL_HOST": os.getenv("EMAIL_HOST", global_settings.EMAIL_HOST), + "EMAIL_HOST_PASSWORD": os.getenv( + "EMAIL_HOST_PASSWORD", global_settings.EMAIL_HOST_PASSWORD + ), + "EMAIL_HOST_USER": os.getenv("EMAIL_HOST_USER", global_settings.EMAIL_HOST_USER), + "EMAIL_PORT": os.getenv("EMAIL_PORT", global_settings.EMAIL_PORT), + "EMAIL_SUBJECT_PREFIX": os.getenv( + "EMAIL_SUBJECT_PREFIX", global_settings.EMAIL_SUBJECT_PREFIX + ), + "EMAIL_SSL_CERTFILE": os.getenv( + "EMAIL_SSL_CERTFILE", global_settings.EMAIL_SSL_CERTFILE + ), + "EMAIL_SSL_KEYFILE": os.getenv( + "EMAIL_SSL_KEYFILE", global_settings.EMAIL_SSL_KEYFILE + ), + "EMAIL_TIMEOUT": os.getenv("EMAIL_TIMEOUT", global_settings.EMAIL_TIMEOUT), + "EMAIL_USE_LOCALTIME": os.getenv( + "EMAIL_USE_LOCALTIME", global_settings.EMAIL_USE_LOCALTIME + ), + "EMAIL_USE_SSL": os.getenv("EMAIL_USE_SSL", global_settings.EMAIL_USE_SSL), + "EMAIL_USE_TLS": os.getenv("EMAIL_USE_TLS", global_settings.EMAIL_USE_TLS), "LOGGING": { "version": 1, "disable_existing_loggers": False, @@ -28,12 +53,13 @@ }, "root": { "handlers": ["console"], - "level": os.getenv("EMAIL_RELAY_LOG_LEVEL", "INFO"), + "level": os.getenv("LOG_LEVEL", "INFO"), }, }, "INSTALLED_APPS": [ "email_relay", ], + "SERVER_EMAIL": os.getenv("SERVER_EMAIL", global_settings.SERVER_EMAIL), } if not DEBUG: SETTINGS["DATABASES"]["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True From 4e7beecc0a3485af0d1d84850d1298b8dd577406 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 21:34:57 -0500 Subject: [PATCH 25/43] fix router name in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 745c4fc..068e578 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,12 @@ DJANGO_EMAIL_RELAY = { } ``` -4. Add the `EmailRelayDatabaseRouter` to your `DATABASE_ROUTERS` setting: +4. Add the `EmailDatabaseRouter` to your `DATABASE_ROUTERS` setting: ```python DATABASE_ROUTERS = [ ... - 'email_relay.db.EmailRelayDatabaseRouter', + 'email_relay.db.EmailDatabaseRouter', ... ] ``` From ef300a20330b5b61b100f1bc1ec3285654ee86e4 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 21:35:14 -0500 Subject: [PATCH 26/43] move version to pyproject.toml --- pyproject.toml | 5 +---- src/email_relay/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a1d9fad..a596948 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,12 +32,12 @@ dependencies = [ "dj-database-url", ] description = "A Django app for relaying email to an external service via a database." -dynamic = ["version"] keywords = [] license = "MIT" name = "django-email-relay" readme = "README.md" requires-python = ">=3.8" +version = "0.1.0" [project.optional-dependencies] dev = [ @@ -77,9 +77,6 @@ exclude = [ [tool.hatch.build.targets.wheel] packages = ["src/email_relay"] -[tool.hatch.version] -path = "src/email_relay/__init__.py" - [tool.black] exclude = ''' /( diff --git a/src/email_relay/__init__.py b/src/email_relay/__init__.py index 8b1c958..e299119 100644 --- a/src/email_relay/__init__.py +++ b/src/email_relay/__init__.py @@ -1,3 +1,5 @@ from __future__ import annotations -__version__ = "2023.0" +import importlib.metadata + +__version__ = importlib.metadata.version("email_relay") From 5c983a943857e2e887b6ee51ed3cc419432ce45b Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 21:41:19 -0500 Subject: [PATCH 27/43] add changelog --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f886f01 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project attempts to adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- An email backend that stores emails in a database ala a Message model rather than sending them via SMTP or other means +- A database backend that routes writes to the Message model to a separate database +- A Message model that stores the contents of an email +- A relay service intended to be run separately, either as a standalone Docker image or as a management command within a Django project +- Initial documentation (README.md) +- Initial tests +- Initial CI/CD (GitHub Actions) + +[unreleased]: https://github.com/westerveltco/django-email-relay/compare/HEAD...HEAD From 8639a1919850868495cf8414afa4eeb6f75da011 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 21:45:12 -0500 Subject: [PATCH 28/43] add py.typed file, aspirationally --- src/email_relay/py.typed | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/email_relay/py.typed diff --git a/src/email_relay/py.typed b/src/email_relay/py.typed new file mode 100644 index 0000000..e69de29 From fff47a0f841dcbb25e9268aadf408b768e0a2b32 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 21 Sep 2023 21:51:41 -0500 Subject: [PATCH 29/43] fix backend name in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 068e578..c48448a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ INSTALLED_APPS = [ ] ``` -3. Add the `EmailRelayBackend` to your `EMAIL_BACKEND` setting: +3. Add the `RelayDatabaseEmailBackend` to your `EMAIL_BACKEND` setting: ```python EMAIL_BACKEND = 'email_relay.backend.RelayDatabaseEmailBackend' From 355f8537d49eab415c5a1425d416fe298073d83e Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Fri, 22 Sep 2023 10:14:52 -0500 Subject: [PATCH 30/43] change test workflow trigger (#14) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 306ea83..06cc825 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ name: test on: push: branches: main - pull_request_target: + pull_request: concurrency: group: test-${{ github.head_ref }} From 14bbdac58c9ace93bf4eb9b8f99ae1074ad7a5d3 Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Fri, 22 Sep 2023 10:17:13 -0500 Subject: [PATCH 31/43] move version back to package (#13) --- pyproject.toml | 5 ++++- src/email_relay/__init__.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a596948..a1d9fad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,12 +32,12 @@ dependencies = [ "dj-database-url", ] description = "A Django app for relaying email to an external service via a database." +dynamic = ["version"] keywords = [] license = "MIT" name = "django-email-relay" readme = "README.md" requires-python = ">=3.8" -version = "0.1.0" [project.optional-dependencies] dev = [ @@ -77,6 +77,9 @@ exclude = [ [tool.hatch.build.targets.wheel] packages = ["src/email_relay"] +[tool.hatch.version] +path = "src/email_relay/__init__.py" + [tool.black] exclude = ''' /( diff --git a/src/email_relay/__init__.py b/src/email_relay/__init__.py index e299119..f11ec6a 100644 --- a/src/email_relay/__init__.py +++ b/src/email_relay/__init__.py @@ -1,5 +1,3 @@ from __future__ import annotations -import importlib.metadata - -__version__ = importlib.metadata.version("email_relay") +__version__ = "0.1.0" From c28779b9974b849578afdd1a5887b25ab81145b9 Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Fri, 22 Sep 2023 10:19:17 -0500 Subject: [PATCH 32/43] remove Django 4.0 (#12) --- .github/workflows/test.yml | 2 +- README.md | 6 +++--- noxfile.py | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06cc825..24ba6cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - django-version: ['3.2', '4.0', '4.1', '4.2', 'main'] + django-version: ['3.2', '4.1', '4.2', 'main'] steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index c48448a..7521756 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ [![PyPI](https://img.shields.io/pypi/v/django-email-relay)](https://pypi.org/project/django-email-relay/) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/django-email-relay) -![Django Version](https://img.shields.io/badge/django-3.2%20%7C%204.0%20%7C%204.1%20%7C%204.2-%2344B78B?labelColor=%23092E20) +![Django Version](https://img.shields.io/badge/django-3.2%20%7C%204.1%20%7C%204.2-%2344B78B?labelColor=%23092E20) - + `django-email-relay` enables Django projects without direct access to a preferred SMTP server to use that server for email dispatch. @@ -20,7 +20,7 @@ Why opt for this setup? ## Requirements - Python 3.8, 3.9, 3.10, 3.11, or 3.12 -- Django 3.2, 4.0, 4.1, or 4.2 +- Django 3.2, 4.1, or 4.2 - PostgreSQL (for provided Docker image) ## Installation diff --git a/noxfile.py b/noxfile.py index 35f752e..862815e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -11,12 +11,11 @@ PY_DEFAULT = PY38 DJ32 = "3.2" -DJ40 = "4.0" DJ41 = "4.1" DJ42 = "4.2" DJMAIN = "main" DJMAIN_MIN_PY = PY310 -DJ_VERSIONS = [DJ32, DJ40, DJ41, DJ42, DJMAIN] +DJ_VERSIONS = [DJ32, DJ41, DJ42, DJMAIN] DJ_DEFAULT = DJ32 From d99f6ec2dfa9a2c6a072d7ebc97bdebab3daca23 Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Mon, 25 Sep 2023 10:38:58 -0500 Subject: [PATCH 33/43] add support for user settings from environ (#15) * add support for user settings from environ * make mypy happy --- service.py | 77 ++++++++++++++++++++++--------------------- tests/test_service.py | 48 +++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 37 deletions(-) create mode 100644 tests/test_service.py diff --git a/service.py b/service.py index 8859f47..dfa8ab0 100644 --- a/service.py +++ b/service.py @@ -1,48 +1,43 @@ from __future__ import annotations import os +from typing import Any import dj_database_url import django -from django.conf import global_settings from django.conf import settings from django.core.management import call_command -DEBUG = os.getenv("EMAIL_RELAY_DEBUG", False) -SETTINGS = { +def get_user_settings_from_env() -> dict[str, Any]: + all_env_vars = {k: v for k, v in os.environ.items()} + return env_vars_to_nested_dict(all_env_vars) + + +def env_vars_to_nested_dict(env_vars: dict[str, Any]) -> dict[str, Any]: + config: dict[str, Any] = {} + for key, value in env_vars.items(): + keys = key.split("__") + d = config + for k in keys[:-1]: + d = d.setdefault(k, {}) + d[keys[-1]] = value + return config + + +def merge_dicts(dict1: dict[str, Any], dict2: dict[str, Any]) -> dict[str, Any]: + for key, value in dict2.items(): + if isinstance(value, dict) and isinstance(dict1.get(key), dict): + merge_dicts(dict1[key], value) + else: + dict1[key] = value + return dict1 + + +default_settings = { "DATABASES": { - "default": dj_database_url.parse( - os.getenv("DATABASE_URL", ""), - conn_max_age=600, # 10 minutes - conn_health_checks=True, - ), + "default": dj_database_url.parse(os.getenv("DATABASE_URL", "sqlite://:memory:")) }, - "DEBUG": DEBUG, - "DEFAULT_FROM_EMAIL": os.getenv( - "DEFAULT_FROM_EMAIL", global_settings.DEFAULT_FROM_EMAIL - ), - "EMAIL_HOST": os.getenv("EMAIL_HOST", global_settings.EMAIL_HOST), - "EMAIL_HOST_PASSWORD": os.getenv( - "EMAIL_HOST_PASSWORD", global_settings.EMAIL_HOST_PASSWORD - ), - "EMAIL_HOST_USER": os.getenv("EMAIL_HOST_USER", global_settings.EMAIL_HOST_USER), - "EMAIL_PORT": os.getenv("EMAIL_PORT", global_settings.EMAIL_PORT), - "EMAIL_SUBJECT_PREFIX": os.getenv( - "EMAIL_SUBJECT_PREFIX", global_settings.EMAIL_SUBJECT_PREFIX - ), - "EMAIL_SSL_CERTFILE": os.getenv( - "EMAIL_SSL_CERTFILE", global_settings.EMAIL_SSL_CERTFILE - ), - "EMAIL_SSL_KEYFILE": os.getenv( - "EMAIL_SSL_KEYFILE", global_settings.EMAIL_SSL_KEYFILE - ), - "EMAIL_TIMEOUT": os.getenv("EMAIL_TIMEOUT", global_settings.EMAIL_TIMEOUT), - "EMAIL_USE_LOCALTIME": os.getenv( - "EMAIL_USE_LOCALTIME", global_settings.EMAIL_USE_LOCALTIME - ), - "EMAIL_USE_SSL": os.getenv("EMAIL_USE_SSL", global_settings.EMAIL_USE_SSL), - "EMAIL_USE_TLS": os.getenv("EMAIL_USE_TLS", global_settings.EMAIL_USE_TLS), "LOGGING": { "version": 1, "disable_existing_loggers": False, @@ -59,13 +54,21 @@ "INSTALLED_APPS": [ "email_relay", ], - "SERVER_EMAIL": os.getenv("SERVER_EMAIL", global_settings.SERVER_EMAIL), } -if not DEBUG: - SETTINGS["DATABASES"]["default"]["DISABLE_SERVER_SIDE_CURSORS"] = True -if __name__ == "__main__": + +def main() -> int: + user_settings = get_user_settings_from_env() + SETTINGS = merge_dicts(default_settings, user_settings) + settings.configure(**SETTINGS) django.setup() call_command("migrate") call_command("runrelay") + # should never get here, `runrelay` is an infinite loop + # but if it does, exit with 0 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..0306fc9 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from service import env_vars_to_nested_dict +from service import merge_dicts + + +def test_env_vars_to_nested_dict(): + env_vars = { + "DATABASES__default__CONN_MAX_AGE": 600, + "DEBUG": "True", + } + + assert env_vars_to_nested_dict(env_vars) == { + "DATABASES": { + "default": { + "CONN_MAX_AGE": 600, + } + }, + "DEBUG": "True", + } + + +def test_merge_dicts(): + default_settings = { + "DATABASES": { + "default": { + "CONN_MAX_AGE": 600, + } + }, + "DEBUG": False, + } + user_settings = { + "DATABASES": { + "default": { + "CONN_MAX_AGE": 300, + } + }, + "DEBUG": True, + } + + assert merge_dicts(default_settings, user_settings) == { + "DATABASES": { + "default": { + "CONN_MAX_AGE": 300, + } + }, + "DEBUG": True, + } From b39f1eba646e491ef47e26c404a0b0a2ed600deb Mon Sep 17 00:00:00 2001 From: Josh Thomas Date: Mon, 25 Sep 2023 11:11:21 -0500 Subject: [PATCH 34/43] add more logging to relay (#19) --- .../management/commands/runrelay.py | 10 ++++++---- src/email_relay/relay.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/email_relay/management/commands/runrelay.py b/src/email_relay/management/commands/runrelay.py index d3dbb3a..c306871 100644 --- a/src/email_relay/management/commands/runrelay.py +++ b/src/email_relay/management/commands/runrelay.py @@ -14,14 +14,16 @@ class Command(BaseCommand): def handle(self, *args, **options): + logger.info("starting relay") while True: while ( not Message.objects.queued().exists() and not Message.objects.deferred().exists() ): - logger.debug( - f"sleeping for {app_settings.EMPTY_QUEUE_SLEEP} seconds before checking queue again" - ) - time.sleep(app_settings.EMPTY_QUEUE_SLEEP) + msg = "queue is empty" + if app_settings.EMPTY_QUEUE_SLEEP > 0: + msg += f", sleeping for {app_settings.EMPTY_QUEUE_SLEEP} seconds before checking queue again" + time.sleep(app_settings.EMPTY_QUEUE_SLEEP) + logger.debug(msg) send_all() diff --git a/src/email_relay/relay.py b/src/email_relay/relay.py index 1c0d790..6ccbed7 100644 --- a/src/email_relay/relay.py +++ b/src/email_relay/relay.py @@ -16,6 +16,8 @@ def send_all(): + logger.info("sending emails") + connection = get_connection(backend=app_settings.EMAIL_BACKEND) counts = { @@ -29,8 +31,13 @@ def send_all(): Message.objects.deferred().prioritized(), ) ) + logger.debug(f"found {len(message_batch)} messages to send") if app_settings.EMAIL_MAX_BATCH is not None: + msg = f"max batch size is {app_settings.EMAIL_MAX_BATCH}" + if len(message_batch) > app_settings.EMAIL_MAX_BATCH: + msg += ", truncating" + logger.debug(msg) message_batch = message_batch[: app_settings.EMAIL_MAX_BATCH] for message in message_batch: @@ -50,6 +57,7 @@ def send_all(): if email is not None: email.connection = connection email.send() + logger.debug(f"sent message {message.id}") message.mark_sent() counts["sent"] += 1 else: @@ -67,8 +75,12 @@ def send_all(): socket_error, ), ): + logger.debug(f"deferring message {message.id} due to {err}") message.defer(log=str(err)) if message.retry_count >= app_settings.EMAIL_MAX_RETRIES: + logger.warning( + f"max retries reached, marking message {message.id} as failed" + ) message.fail(log=str(err)) connection = None counts["deferred"] += 1 @@ -79,7 +91,15 @@ def send_all(): app_settings.EMAIL_MAX_DEFERRED is not None and counts["deferred"] >= app_settings.EMAIL_MAX_DEFERRED ): + logger.debug( + f"max deferred emails reached ({app_settings.EMAIL_MAX_DEFERRED}), stopping" + ) break if app_settings.EMAIL_THROTTLE > 0: + logger.debug( + f"throttling enabled, sleeping for {app_settings.EMAIL_THROTTLE} seconds" + ) time.sleep(app_settings.EMAIL_THROTTLE) + + logger.info(f"sent {counts['sent']} emails, deferred {counts['deferred']} emails") From 13f32ced729cf3ddf885dc48ffaa1988d7f0125f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:15:24 -0500 Subject: [PATCH 35/43] :robot: Bump docker/build-push-action from 4 to 5 (#16) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb7dfb2..6eb246f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,7 +79,7 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and publish Docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true From 479f233e5f7ecad320c5bc47dab8384032416e78 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:15:36 -0500 Subject: [PATCH 36/43] :robot: Bump docker/login-action from 2 to 3 (#17) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6eb246f..93e41a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} From 0dbb57443046851459d5fa8bb0e7eed26777dbaa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:15:49 -0500 Subject: [PATCH 37/43] :robot: Bump docker/setup-buildx-action from 2 to 3 (#18) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93e41a0..d45434b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: From ee6a3e5d0bbeca78f21fd6a126d822f39c7c5595 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:16:31 -0500 Subject: [PATCH 38/43] :robot: [pre-commit.ci] pre-commit autoupdate (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.14.1 → 1.15.0](https://github.com/adamchainz/django-upgrade/compare/1.14.1...1.15.0) - [github.com/astral-sh/ruff-pre-commit: v0.0.290 → v0.0.291](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.290...v0.0.291) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de81184..760a1d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,13 +11,13 @@ repos: - id: check-yaml - repo: https://github.com/adamchainz/django-upgrade - rev: 1.14.1 + rev: 1.15.0 hooks: - id: django-upgrade args: [--target-version, "4.2"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.290 + rev: v0.0.291 hooks: - id: ruff alias: autoformat From f0b95d3c2d752a1f7a49730e3fb508fd388c3d2e Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 15:51:27 -0500 Subject: [PATCH 39/43] :tractor: Run pytest-cov --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24ba6cc..d1511b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: # https://hynek.me/articles/ditch-codecov-python/ - name: Run tests run: | - coverage run -m pytest + pytest -vv python -m coverage html --skip-covered --skip-empty python -m coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY python -m coverage report --fail-under=30 From 92f981c785d79ee248106c02a87f147292fe8037 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 15:58:41 -0500 Subject: [PATCH 40/43] :arrow_up: Bumps coverage to check to >50% --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d1511b2..6e16e41 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,4 +91,4 @@ jobs: pytest -vv python -m coverage html --skip-covered --skip-empty python -m coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY - python -m coverage report --fail-under=30 + python -m coverage report --fail-under=50 From 674cdf812cd7df5217c8e58ba29541facd1ebf73 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 15:59:14 -0500 Subject: [PATCH 41/43] :gear: Run normal pytest --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e16e41..88ead91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: # https://hynek.me/articles/ditch-codecov-python/ - name: Run tests run: | - pytest -vv + pytest python -m coverage html --skip-covered --skip-empty python -m coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY python -m coverage report --fail-under=50 From 2e5b499588ecc22b6c18d1bb12089449b8e4452d Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 16:34:16 -0500 Subject: [PATCH 42/43] :tractor: Adds python -m to all the things --- .github/workflows/test.yml | 2 +- Justfile | 5 +---- noxfile.py | 10 +++++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88ead91..58bd467 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: # https://hynek.me/articles/ditch-codecov-python/ - name: Run tests run: | - pytest + python -m pytest python -m coverage html --skip-covered --skip-empty python -m coverage report | sed 's/^/ /' >> $GITHUB_STEP_SUMMARY python -m coverage report --fail-under=50 diff --git a/Justfile b/Justfile index bf5af13..a89f420 100644 --- a/Justfile +++ b/Justfile @@ -48,10 +48,7 @@ test: python -m nox --reuse-existing-virtualenvs coverage: - rm -rf .coverage htmlcov - pytest -vv - python -m coverage html --skip-empty # --skip-covered - python -m coverage report --fail-under=100 + python -m nox --reuse-existing-virtualenvs --session "coverage" types: python -m mypy . diff --git a/noxfile.py b/noxfile.py index 862815e..85b048b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -49,7 +49,15 @@ def tests(session, django): else: session.install(f"django=={django}") - session.run("pytest", "-n", "auto", "--dist", "loadfile") + session.run("python", "-m", "pytest", "-n", "auto", "--dist", "loadfile") + + +@nox.session +def coverage(session): + session.install(".[dev]") + session.run("python", "-m", "pytest") + session.run("python", "-m", "coverage", "html", "--skip-covered", "--skip-empty") + session.run("python", "-m", "coverage", "report", "--fail-under=50") @nox.session From 3a28784014322366ee271a19d03ded853d8bc812 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Mon, 25 Sep 2023 16:35:09 -0500 Subject: [PATCH 43/43] :tractor: Updates nox entrypoint too --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58bd467..632d2eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: - name: Run tests run: | - nox --session "tests-${{ matrix.python-version }}(django='${{ matrix.django-version }}')" + python -m nox --session "tests-${{ matrix.python-version }}(django='${{ matrix.django-version }}')" tests: runs-on: ubuntu-latest