Fixes for Debug Web

django_lightweight_queue/app_settings.py

@@ -1,52 +1,202 @@
-from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence
+from typing import Any, Dict, Union, TypeVar, Callable, Optional, Sequence
 
-from django.conf import settings
+from django.conf import settings as django_settings
 
 from . import constants
 from .types import Logger, QueueName
 
 T = TypeVar('T')
 
 
-def setting(suffix: str, default: T) -> T:
-    attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix)
-    return getattr(settings, attr_name, default)
+class Settings:
+    def _get(self, suffix: str, default: T) -> T:
+        attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix)
+        return getattr(django_settings, attr_name, default)
 
+    # adjustable values at runtime
+    _workers = None
+    _backend = None
+    _logger_factory = None
+    _backend_overrides = None
+    _middleware = None
+    _ignore_apps = None
+    _redis_host = None
+    _redis_port = None
+    _redis_password = None
+    _redis_prefix = None
+    _enable_prometheus = None
+    _prometheus_start_port = None
+    _atomic_jobs = None
+    _site_url = None
 
-WORKERS = setting('WORKERS', {})  # type: Dict[QueueName, int]
-BACKEND = setting(
-    'BACKEND',
-    'django_lightweight_queue.backends.synchronous.SynchronousBackend',
-)  # type: str
+    def get_empty_dict(self) -> Dict[Any, Any]:
+        """
+        Declare dummy type to make mypy happy.
 
-LOGGER_FACTORY = setting(
-    'LOGGER_FACTORY',
-    'logging.getLogger',
-)  # type: Union[str, Callable[[str], Logger]]
+        Mypy cannot handle types changing after instantiation, which is
+        exactly what happens to any setting that is a dict. This helper
+        method works around https://github.com/python/mypy/issues/6463
+        and makes mypy happy -- at no point will we actually return
+        Dict[Any, Any] because between the time that mypy reads it and
+        the time that we actually need it, it will have been populated
+        with the value that we actually need it to be.
+        """
+        return {}
 
-# Allow per-queue overrides of the backend.
-BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {})  # type: Mapping[QueueName, str]
+    @property
+    def WORKERS(self) -> Dict[QueueName, int]:
+        if not self._workers:
+            self._workers = self._get('WORKERS', self.get_empty_dict())
+        return self._workers
 
-MIDDLEWARE = setting('MIDDLEWARE', (
-    'django_lightweight_queue.middleware.logging.LoggingMiddleware',
-    'django_lightweight_queue.middleware.transaction.TransactionMiddleware',
-))  # type: Sequence[str]
+    @WORKERS.setter
+    def WORKERS(self, value):
+        self._workers = value
 
-# Apps to ignore when looking for tasks. Apps must be specified as the dotted
-# name used in `INSTALLED_APPS`. This is expected to be useful when you need to
-# have a file called `tasks.py` within an app, but don't want
-# django-lightweight-queue to import that file.
-# Note: this _doesn't_ prevent tasks being registered from these apps.
-IGNORE_APPS = setting('IGNORE_APPS', ())  # type: Sequence[str]
+    @property
+    def BACKEND(self) -> str:
+        if not self._backend:
+            self._backend = self._get(
+                'BACKEND',
+                'django_lightweight_queue.backends.synchronous.SynchronousBackend',
+            )
+        return self._backend
 
-# Backend-specific settings
-REDIS_HOST = setting('REDIS_HOST', '127.0.0.1')  # type: str
-REDIS_PORT = setting('REDIS_PORT', 6379)  # type: int
-REDIS_PASSWORD = setting('REDIS_PASSWORD', None)  # type: Optional[str]
-REDIS_PREFIX = setting('REDIS_PREFIX', '')  # type: str
+    @BACKEND.setter
+    def BACKEND(self, value):
+        self._backend = value
 
-ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False)  # type: bool
-# Workers will export metrics on this port, and ports following it
-PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300)  # type: int
+    @property
+    def LOGGER_FACTORY(self) -> Union[str, Callable[[str], Logger]]:
+        if not self._logger_factory:
+            self._logger_factory = self._get(
+                'LOGGER_FACTORY',
+                'logging.getLogger',
+            )
+        return self._logger_factory
 
-ATOMIC_JOBS = setting('ATOMIC_JOBS', True)  # type: bool
+    @LOGGER_FACTORY.setter
+    def LOGGER_FACTORY(self, value):
+        self._logger_factory = value
+
+    @property
+    def BACKEND_OVERRIDES(self) -> Dict[QueueName, str]:
+        # Allow per-queue overrides of the backend.
+        if not self._backend_overrides:
+            self._backend_overrides = self._get('BACKEND_OVERRIDES', self.get_empty_dict())
+        return self._backend_overrides
+
+    @BACKEND_OVERRIDES.setter
+    def BACKEND_OVERRIDES(self, value):
+        self._backend_overrides = value
+
+    @property
+    def MIDDLEWARE(self) -> Sequence[str]:
+        if not self._middleware:
+            self._middleware = self._get('MIDDLEWARE', (
+                'django_lightweight_queue.middleware.logging.LoggingMiddleware',
+            ))
+        return self._middleware
+
+    @MIDDLEWARE.setter
+    def MIDDLEWARE(self, value):
+        self._middleware = value
+
+    @property
+    def IGNORE_APPS(self) -> Sequence[str]:
+        # Apps to ignore when looking for tasks. Apps must be specified as the dotted
+        # name used in `INSTALLED_APPS`. This is expected to be useful when you need to
+        # have a file called `tasks.py` within an app, but don't want
+        # django-lightweight-queue to import that file.
+        # Note: this _doesn't_ prevent tasks being registered from these apps.
+        if not self._ignore_apps:
+            self._ignore_apps = self._get('IGNORE_APPS', ())
+        return self._ignore_apps
+
+    @IGNORE_APPS.setter
+    def IGNORE_APPS(self, value):
+        self._ignore_apps = value
+
+    @property
+    def REDIS_HOST(self) -> str:
+        if not self._redis_host:
+            self._redis_host = self._get('REDIS_HOST', '127.0.0.1')
+        return self._redis_host
+
+    @REDIS_HOST.setter
+    def REDIS_HOST(self, value):
+        self._redis_host = value
+
+    @property
+    def REDIS_PORT(self) -> int:
+        if not self._redis_port:
+            self._redis_port = self._get('REDIS_PORT', 6379)
+        return self._redis_port
+
+    @REDIS_PORT.setter
+    def REDIS_PORT(self, value):
+        self._redis_port = value
+
+    @property
+    def REDIS_PASSWORD(self) -> Optional[str]:
+        if not self._redis_password:
+            self._redis_password = self._get('REDIS_PASSWORD', None)
+        return self._redis_password
+
+    @REDIS_PASSWORD.setter
+    def REDIS_PASSWORD(self, value):
+        self._redis_password = value
+
+    @property
+    def REDIS_PREFIX(self) -> str:
+        if not self._redis_prefix:
+            self._redis_prefix = self._get('REDIS_PREFIX', '')
+        return self._redis_prefix
+
+    @REDIS_PREFIX.setter
+    def REDIS_PREFIX(self, value):
+        self._redis_prefix = value
+
+    @property
+    def ENABLE_PROMETHEUS(self) -> bool:
+        if not self._enable_prometheus:
+            self._enable_prometheus = self._get('ENABLE_PROMETHEUS', False)
+        return self._enable_prometheus
+
+    @ENABLE_PROMETHEUS.setter
+    def ENABLE_PROMETHEUS(self, value):
+        self._enable_prometheus = value
+
+    @property
+    def PROMETHEUS_START_PORT(self) -> int:
+        # Workers will export metrics on this port, and ports following it
+        if not self._prometheus_start_port:
+            self._prometheus_start_port = self._get('PROMETHEUS_START_PORT', 9300)
+        return self._prometheus_start_port
+
+    @PROMETHEUS_START_PORT.setter
+    def PROMETHEUS_START_PORT(self, value):
+        self._prometheus_start_port = value
+
+    @property
+    def ATOMIC_JOBS(self) -> bool:
+        if not self._atomic_jobs:
+            self._atomic_jobs = self._get('ATOMIC_JOBS', True)
+        return self._atomic_jobs
+
+    @ATOMIC_JOBS.setter
+    def ATOMIC_JOBS(self, value):
+        self._atomic_jobs = value
+
+    @property
+    def SITE_URL(self) -> str:
+        if not self._site_url:
+            self._site_url = self._get('SITE_URL', "http://localhost:8000")
+        return self._site_url
+
+    @SITE_URL.setter
+    def SITE_URL(self, value):
+        self._site_url = value
+
+
+app_settings = Settings()

django_lightweight_queue/backends/debug_web.py

@@ -1,11 +1,11 @@
 import urllib.parse
 
-from django.conf import settings
 from django.shortcuts import reverse
 
 from ..job import Job
 from .base import BaseBackend
 from ..types import QueueName, WorkerNumber
+from ..app_settings import app_settings
 
 
 class DebugWebBackend(BaseBackend):
@@ -20,9 +20,9 @@ class DebugWebBackend(BaseBackend):
     """
 
     def enqueue(self, job: Job, queue: QueueName) -> None:
-        path = reverse('django-lightweight-queue:debug-run')
+        path = reverse('django_lightweight_queue:debug-run')
         query_string = urllib.parse.urlencode({'job': job.to_json()})
-        url = "{}{}?{}".format(settings.SITE_URL, path, query_string)
+        url = "{}{}?{}".format(app_settings.SITE_URL, path, query_string)
         print(url)
 
     def dequeue(self, queue: QueueName, worker_num: WorkerNumber, timeout: float) -> None:

django_lightweight_queue/backends/redis.py

@@ -3,11 +3,11 @@
 
 import redis
 
-from .. import app_settings
 from ..job import Job
 from .base import BackendWithPauseResume
 from ..types import QueueName, WorkerNumber
 from ..utils import block_for_time
+from ..app_settings import app_settings
 
 
 class RedisBackend(BackendWithPauseResume):

django_lightweight_queue/backends/reliable_redis.py

@@ -3,11 +3,11 @@
 
 import redis
 
-from .. import app_settings
 from ..job import Job
 from .base import BackendWithDeduplicate, BackendWithPauseResume
 from ..types import QueueName, WorkerNumber
 from ..utils import block_for_time, get_worker_numbers
+from ..app_settings import app_settings
 from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER
 
 # Work around https://github.com/python/mypy/issues/9914. Name needs to match

django_lightweight_queue/exposition.py

@@ -6,8 +6,8 @@
 
 from prometheus_client.exposition import MetricsHandler
 
-from . import app_settings
 from .types import QueueName, WorkerNumber
+from .app_settings import app_settings
 
 
 def get_config_response(

django_lightweight_queue/management/commands/queue_configuration.py

@@ -2,8 +2,8 @@
 
 from django.core.management.base import BaseCommand, CommandParser
 
-from ... import app_settings
 from ...utils import get_backend, get_queue_counts, load_extra_config
+from ...app_settings import app_settings
 from ...cron_scheduler import get_cron_config
 
 

django_lightweight_queue/runner.py

@@ -4,10 +4,10 @@
 import subprocess
 from typing import Dict, Tuple, Callable, Optional
 
-from . import app_settings
 from .types import Logger, QueueName, WorkerNumber
 from .utils import get_backend, set_process_title
 from .exposition import metrics_http_server
+from .app_settings import app_settings
 from .machine_types import Machine
 from .cron_scheduler import (
     CronScheduler,

django_lightweight_queue/task.py

@@ -12,10 +12,10 @@
     Optional,
 )
 
-from . import app_settings
 from .job import Job
 from .types import QueueName
 from .utils import get_backend, contribute_implied_queue_name
+from .app_settings import app_settings
 
 TCallable = TypeVar('TCallable', bound=Callable[..., Any])
 

django_lightweight_queue/utils.py

@@ -21,8 +21,9 @@
 from django.core.exceptions import MiddlewareNotUsed
 from django.utils.module_loading import module_has_submodule
 
-from . import constants, app_settings
+from . import constants
 from .types import Logger, QueueName, WorkerNumber
+from .app_settings import app_settings
 
 if TYPE_CHECKING:
     from .backends.base import BaseBackend

django_lightweight_queue/worker.py

@@ -12,9 +12,9 @@
 
 from django.db import connections, transaction
 
-from . import app_settings
 from .types import QueueName, WorkerNumber
 from .utils import get_logger, get_backend, set_process_title
+from .app_settings import app_settings
 from .backends.base import BaseBackend
 
 if app_settings.ENABLE_PROMETHEUS:

tests/test_pause_resume.py

@@ -52,7 +52,7 @@ def setUp(self) -> None:
     # Can't use override_settings due to the copying of the settings values into
     # module values at startup.
     @mock.patch(
-        'django_lightweight_queue.app_settings.BACKEND',
+        'django_lightweight_queue.app_settings.Settings.BACKEND',
         new='django_lightweight_queue.backends.redis.RedisBackend',
     )
     def test_pause_resume(self) -> None:

tests/test_reliable_redis_backend.py

@@ -50,7 +50,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]:
             'django_lightweight_queue.utils._accepting_implied_queues',
             new=False,
         ), unittest.mock.patch.dict(
-            'django_lightweight_queue.app_settings.WORKERS',
+            'django_lightweight_queue.app_settings.app_settings.WORKERS',
             workers,
         ):
             yield

tests/test_task.py

@@ -30,7 +30,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]:
             'django_lightweight_queue.utils._accepting_implied_queues',
             new=False,
         ), unittest.mock.patch.dict(
-            'django_lightweight_queue.app_settings.WORKERS',
+            'django_lightweight_queue.app_settings.Settings.WORKERS',
             workers,
         ):
             yield
@@ -54,7 +54,7 @@ def mocked_get_path(path: str) -> Any:
             return get_path(path)
 
         patch = mock.patch(
-            'django_lightweight_queue.app_settings.BACKEND',
+            'django_lightweight_queue.app_settings.Settings.BACKEND',
             new='test-backend',
         )
         patch.start()