Minor fixes for cache manager and processor

.circleci/config.yml

@@ -59,7 +59,7 @@ jobs:
       - run:
           name: Verify requirements.txt file
           command: |
-            docker run --rm local/socorro_app shell ./bin/run_verify_reqs.sh
+            docker run --rm local/socorro_app shell ./bin/verify_reqs.sh
 
       - run:
           name: Lint

.github/workflows/build-and-push.yml

@@ -41,7 +41,7 @@ jobs:
           docker compose images
       - name: Verify requirements.txt contains correct dependencies
         run: |
-          docker run --rm local/socorro_app shell ./bin/run_verify_reqs.sh
+          docker run --rm local/socorro_app shell ./bin/verify_reqs.sh
       - name: Run lint check
         run: |
           docker run --rm local/socorro_app shell ./bin/lint.sh

bin/entrypoint.sh

@@ -29,13 +29,13 @@ shift
 
 case ${SERVICE} in
 processor)  ## Run processor service
-    /app/bin/run_processor.sh "$@"
+    /app/bin/run_service_processor.sh "$@"
     ;;
 crontabber)  ## Run crontabber service
-    /app/bin/run_crontabber.sh "$@"
+    /app/bin/run_service_crontabber.sh "$@"
     ;;
 webapp)  ## Run webapp service
-    /app/bin/run_webapp.sh "$@"
+    /app/bin/run_service_webapp.sh "$@"
     ;;
 symbolsserver)  ## Runs a local symbols server
     /app/bin/run_symbolsserver.sh

bin/run_processor_cache_manager.sh → bin/run_cache_manager.sh

@@ -4,7 +4,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Usage: bin/run_processor_cache_manager.sh
+# Usage: bin/run_cache_manager.sh
 #
 # Runs the processor cache manager.
 #

bin/run_processor.sh

@@ -6,18 +6,13 @@
 
 # Usage: bin/run_processor.sh
 #
-# Runs processor process manager.
+# Runs the processor process.
 #
 # Note: This should be called from inside a container.
 
 set -euo pipefail
 
-PROCESSOR_WORKERS=${PROCESSOR_WORKERS:-"1"}
+export PROCESS_NAME=processor
 
-# Run honcho with PROCESSOR_WORKERS number of processor worker processes
-honcho \
-    --procfile /app/processor/Procfile \
-    --app-root /app \
-    --no-prefix \
-    start \
-    --concurrency "processor=${PROCESSOR_WORKERS}"
+# Run the processor
+python socorro/processor/processor_app.py

bin/run_crontabber.sh → bin/run_service_crontabber.sh

@@ -4,9 +4,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Usage: bin/run_crontabber.sh
+# Usage: bin/run_service_crontabber.sh
 #
-# Runs cronrun (used to be called crontabber).
+# Runs cronrun (used to be called crontabber) service.
 #
 # Note: This should be called from inside a container.
 

bin/run_service_processor.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Usage: bin/run_service_processor.sh
+#
+# Runs processor service. The processor service is composed of two processes:
+#
+# * processor
+# * cache_manager
+#
+# Note: This should be called from inside a container.
+
+set -euo pipefail
+
+PROCESSOR_WORKERS=${PROCESSOR_WORKERS:-"1"}
+
+# Run honcho with PROCESSOR_WORKERS number of processor worker processes
+honcho \
+    --procfile /app/processor/Procfile \
+    --app-root /app \
+    --no-prefix \
+    start \
+    --concurrency "processor=${PROCESSOR_WORKERS}"

bin/run_webapp.sh → bin/run_service_webapp.sh

@@ -4,9 +4,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Usage: bin/run_webapp.sh [--dev]
+# Usage: bin/run_service_webapp.sh [--dev]
 #
-# Runs the webapp.
+# Runs the webapp service.
 #
 # Use the "--dev" argument to run the webapp in a docker container for
 # local development.

bin/run_verify_reqs.sh → bin/verify_reqs.sh

@@ -4,7 +4,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Usage: bin/run_verify_reqs.sh
+# Usage: bin/verify_reqs.sh
 #
 # Verifies that the requirements file is built by the version of Python that
 # runs in the container.

docs/configuration.rst

@@ -29,8 +29,8 @@ Gunicorn configuration:
 
    https://docs.gunicorn.org/en/stable/settings.html#timeout
 
-   Used in `bin/run_webapp.sh
-   <https://github.com/mozilla-services/socorro/blob/main/bin/run_webapp.sh>`_.
+   Used in `bin/run_service_webapp.sh
+   <https://github.com/mozilla-services/socorro/blob/main/bin/run_service_webapp.sh>`_.
 
 
 .. everett:option:: GUNICORN_WORKERS
@@ -44,8 +44,8 @@ Gunicorn configuration:
 
    http://docs.gunicorn.org/en/stable/design.html#how-many-workers
 
-   Used in `bin/run_webapp.sh
-   <https://github.com/mozilla-services/socorro/blob/main/bin/run_webapp.sh>`_.
+   Used in `bin/run_service_webapp.sh
+   <https://github.com/mozilla-services/socorro/blob/main/bin/run_service_webapp.sh>`_.
 
 
 .. everett:option:: GUNICORN_WORKER_CLASS
@@ -55,8 +55,8 @@ Gunicorn configuration:
 
    https://docs.gunicorn.org/en/stable/settings.html#workers
 
-   Used in `bin/run_webapp.sh
-   <https://github.com/mozilla-services/socorro/blob/main/bin/run_webapp.sh>`_.
+   Used in `bin/run_service_webapp.sh
+   <https://github.com/mozilla-services/socorro/blob/main/bin/run_service_webapp.sh>`_.
 
 
 .. everett:option:: GUNICORN_MAX_REQUESTS
@@ -66,8 +66,8 @@ Gunicorn configuration:
 
    https://docs.gunicorn.org/en/stable/settings.html#workers
 
-   Used in `bin/run_webapp.sh
-   <https://github.com/mozilla-services/socorro/blob/main/bin/run_webapp.sh>`_.
+   Used in `bin/run_service_webapp.sh
+   <https://github.com/mozilla-services/socorro/blob/main/bin/run_service_webapp.sh>`_.
 
 
 .. everett:option:: GUNICORN_MAX_REQUESTS_JITTER
@@ -78,8 +78,8 @@ Gunicorn configuration:
 
    https://docs.gunicorn.org/en/stable/settings.html#workers
 
-   Used in `bin/run_webapp.sh
-   <https://github.com/mozilla-services/socorro/blob/main/bin/run_webapp.sh>`_.
+   Used in `bin/run_service_webapp.sh
+   <https://github.com/mozilla-services/socorro/blob/main/bin/run_service_webapp.sh>`_.
 
 
 Webapp configuration:

docs/service/cron.rst

@@ -11,8 +11,8 @@ Job configuration is in ``webapp/crashstats/cron/__init__.py``.
 
 Code is in ``webapp/crashstats/cron/``.
 
-Run script is ``/app/bin/run_crontabber.sh``. This is an infinite loop that
-runs the ``manage.py cronrun`` command every 5 minutes.
+Run script is ``/app/bin/run_service_crontabber.sh``. This is an infinite loop
+that runs the ``manage.py cronrun`` command every 5 minutes.
 
 
 manage.py cronrun

docs/service/processor.rst

@@ -47,8 +47,8 @@ To run the processor in the local dev environment, do::
   $ docker compose up processor
 
 That will bring up all the services the processor requires to run and start the
-processor using the ``/app/bin/run_processor.sh`` script and the processor
-configuration.
+processor using the ``/app/bin/run_service_processor.sh`` script and the
+processor configuration.
 
 To use tools and also ease debugging in the container, you can run a shell::
 
@@ -69,4 +69,4 @@ Run the docker image using the ``processor`` command. Something like this::
         --volume /data:/data \
         mozilla/socorro_app processor
 
-This runs the ``/app/bin/run_processor.sh`` script.
+This runs the ``/app/bin/run_service_processor.sh`` script.

docs/service/webapp.rst

@@ -6,7 +6,7 @@ Crash Stats Webapp
 
 Code is in ``webapp/``.
 
-Run script is ``/app/bin/run_webapp.sh``.
+Run script is ``/app/bin/run_service_webapp.sh``.
 
 
 Configuration
@@ -144,7 +144,7 @@ Then compile the static assets::
 
 Now run the webapp with ``gunicorn`` and ``DEBUG=False``::
 
-  app@socorro:/app$ DEBUG=False bash bin/run_webapp.sh
+  app@socorro:/app$ DEBUG=False bash bin/run_service_webapp.sh
 
 You will now be able to open ``http://localhost:8000`` on the host and if you
 view the source you see that the minified and concatenated static assets are

processor/Procfile

@@ -1,2 +1,2 @@
-processor: /app/bin/run_processor_process.sh
-processor_cache_manager: /app/bin/run_processor_cache_manager.sh
+processor: /app/bin/run_processor.sh
+cache_manager: /app/bin/run_cache_manager.sh

socorro/mozilla_settings.py

@@ -70,7 +70,7 @@ def _or_none(val):
 LOGGING_LEVEL = _config(
     "LOGGING_LEVEL",
     default="INFO",
-    doc="Default logging level. Should be one of INFO, DEBUG, WARNING, ERROR.",
+    doc="Default logging level. Should be one of DEBUG, INFO, WARNING, ERROR.",
 )
 
 # Markus configuration for metrics
@@ -349,7 +349,7 @@ def cloud_provider_parser(val):
     "CACHE_MANAGER_LOGGING_LEVEL",
     default="INFO",
     doc=(
-        "Default logging level for the cache manager. Should be one of INFO, DEBUG, "
+        "Default logging level for the cache manager. Should be one of DEBUG, INFO, "
         "WARNING, ERROR."
     ),
 )

socorro/processor/cache_manager.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
@@ -15,7 +17,7 @@
 
 To run::
 
-    $ /app/bin/run_processor_cache_manager.sh
+    $ /app/bin/run_cache_manager.sh
 
 """
 
@@ -38,14 +40,19 @@
 from socorro.lib.liblogging import set_up_logging
 
 
-LOGGER = logging.getLogger(__name__)
-REPOROOT_DIR = str(pathlib.Path(__file__).parent.parent.parent)
+# How many errors before we give up and terminate the process
 MAX_ERRORS = 10
-METRICS = markus.get_metrics("processor.cache_manager")
+
+# How many seconds between heartbeats
 HEARTBEAT_INTERVAL = 60
 
+# Metrics client to use
+METRICS = markus.get_metrics("processor.cache_manager")
+
 
 def count_sentry_scrub_error(msg):
+    # NOTE(willkg): we re-use the processor prefix here and differentiate with the
+    # service tag.
     metrics = markus.get_metrics("processor")
     metrics.incr("sentry_scrub_error", value=1, tags=["service:cachemanager"])
 
@@ -68,7 +75,8 @@ def pop_oldest(self):
 
 
 def handle_exception(exctype, value, tb):
-    LOGGER.error(
+    logger = logging.getLogger(__name__)
+    logger.error(
         "unhandled exception. Exiting. "
         + "".join(traceback.format_exception(exctype, value, tb))
     )
@@ -554,12 +562,14 @@ def shutdown(self):
 
 
 def main():
-    from socorro.processor import cache_manager
-
-    app = cache_manager.DiskCacheManager()
+    app = DiskCacheManager()
     app.set_up()
     app.run_loop()
 
 
 if __name__ == "__main__":
-    main()
+    # NOTE(willkg): we need to do this so that the cache_manager logger isn't `__main__`
+    # which causes problems when logging
+    from socorro.processor import cache_manager
+
+    cache_manager.main()