From 8d396f210d882e337682edc39dc2d8ddb2dfbdfa Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 18:27:54 -0400 Subject: [PATCH 01/14] resolve conflicts --- README.md | 25 +++- django_lightweight_queue/app_settings.py | 111 ++++++++++++------ django_lightweight_queue/backends/redis.py | 12 +- .../backends/reliable_redis.py | 12 +- django_lightweight_queue/exposition.py | 6 +- .../commands/queue_configuration.py | 4 +- django_lightweight_queue/runner.py | 6 +- django_lightweight_queue/task.py | 4 +- django_lightweight_queue/utils.py | 23 ++-- django_lightweight_queue/worker.py | 8 +- pyproject.toml | 2 +- tests/test_pause_resume.py | 2 +- tests/test_reliable_redis_backend.py | 4 +- tests/test_task.py | 4 +- 14 files changed, 139 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index abce973..68ecac1 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,19 @@ backends are great candidates for community contributions. ## Basic Usage +Start by adding `django_lightweight_queue` to your `INSTALLED_APPS`: + +```python +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + ..., + "django_lightweight_queue", +] +``` + +After that, define your task in any file you want: + ```python import time from django_lightweight_queue import task @@ -67,12 +80,12 @@ present in the specified file are inherited from the global configuration. There are four built-in backends: -| Backend | Type | Description | -| -------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Synchronous | Development | Executes the task inline, without any actual queuing. | -| Redis | Production | Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. | -| Reliable Redis | Production | Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. | -| Debug Web | Debugging | Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. | +| Backend | Import Location | Type | Description | +| -------------- |:----------------------------------------------------------------------| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Synchronous | django_lightweight_queue.backends.synchronous.SynchronousBackend | Development | Executes the task inline, without any actual queuing. | +| Redis | django_lightweight_queue.backends.redis.RedisBackend | Production | Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. | +| Reliable Redis | django_lightweight_queue.backends.reliable_redis.ReliableRedisBackend | Production | Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. | +| Debug Web | django_lightweight_queue.backends.debug_web.DebugWebBackend | Debugging | Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. | [redis]: https://redis.io/ diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index b4bd19c..24ae5e0 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,6 +1,6 @@ -from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence +from typing import Union, Mapping, TypeVar, Callable, Optional, Sequence -from django.conf import settings +from django.conf import settings as django_settings from . import constants from .types import Logger, QueueName @@ -8,45 +8,86 @@ T = TypeVar('T') -def setting(suffix: str, default: T) -> T: - attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) - return getattr(settings, attr_name, default) +class Settings(): + def _get(self, suffix: str, default: T) -> T: + attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) + return getattr(django_settings, attr_name, default) + # adjustable values at runtime + _backend = None -WORKERS = setting('WORKERS', {}) # type: Dict[QueueName, int] -BACKEND = setting( - 'BACKEND', - 'django_lightweight_queue.backends.synchronous.SynchronousBackend', -) # type: str + @property + def WORKERS(self): + return self._get('WORKERS', {}) -LOGGER_FACTORY = setting( - 'LOGGER_FACTORY', - 'logging.getLogger', -) # type: Union[str, Callable[[str], Logger]] + @property + def BACKEND(self): + if not self._backend: + self._backend = self._get( + 'BACKEND', + 'django_lightweight_queue.backends.synchronous.SynchronousBackend', + ) + return self._backend # type: str -# Allow per-queue overrides of the backend. -BACKEND_OVERRIDES = setting('BACKEND_OVERRIDES', {}) # type: Mapping[QueueName, str] + @BACKEND.setter + def BACKEND(self, value): + self._backend = value -MIDDLEWARE = setting('MIDDLEWARE', ( - 'django_lightweight_queue.middleware.logging.LoggingMiddleware', - 'django_lightweight_queue.middleware.transaction.TransactionMiddleware', -)) # type: Sequence[str] + @property + def LOGGER_FACTORY(self): + return self._get( + 'LOGGER_FACTORY', + 'logging.getLogger', + ) # type: Union[str, Callable[[str], Logger]] -# Apps to ignore when looking for tasks. Apps must be specified as the dotted -# name used in `INSTALLED_APPS`. This is expected to be useful when you need to -# have a file called `tasks.py` within an app, but don't want -# django-lightweight-queue to import that file. -# Note: this _doesn't_ prevent tasks being registered from these apps. -IGNORE_APPS = setting('IGNORE_APPS', ()) # type: Sequence[str] + @property + def BACKEND_OVERRIDES(self): + # Allow per-queue overrides of the backend. + return self._get('BACKEND_OVERRIDES', {}) # type: Mapping[QueueName, str] -# Backend-specific settings -REDIS_HOST = setting('REDIS_HOST', '127.0.0.1') # type: str -REDIS_PORT = setting('REDIS_PORT', 6379) # type: int -REDIS_PASSWORD = setting('REDIS_PASSWORD', None) # type: Optional[str] -REDIS_PREFIX = setting('REDIS_PREFIX', '') # type: str + @property + def MIDDLEWARE(self): + return self._get('MIDDLEWARE', ( + 'django_lightweight_queue.middleware.logging.LoggingMiddleware', + )) # type: Sequence[str] -ENABLE_PROMETHEUS = setting('ENABLE_PROMETHEUS', False) # type: bool -# Workers will export metrics on this port, and ports following it -PROMETHEUS_START_PORT = setting('PROMETHEUS_START_PORT', 9300) # type: int + @property + def IGNORE_APPS(self): + # Apps to ignore when looking for tasks. Apps must be specified as the dotted + # name used in `INSTALLED_APPS`. This is expected to be useful when you need to + # have a file called `tasks.py` within an app, but don't want + # django-lightweight-queue to import that file. + # Note: this _doesn't_ prevent tasks being registered from these apps. + return self._get('IGNORE_APPS', ()) # type: Sequence[str] -ATOMIC_JOBS = setting('ATOMIC_JOBS', True) # type: bool + @property + def REDIS_HOST(self): + return self._get('REDIS_HOST', '127.0.0.1') # type: str + + @property + def REDIS_PORT(self): + return self._get('REDIS_PORT', 6379) # type: int + + @property + def REDIS_PASSWORD(self): + return self._get('REDIS_PASSWORD', None) # type: Optional[str] + + @property + def REDIS_PREFIX(self): + return self._get('REDIS_PREFIX', '') # type: str + + @property + def ENABLE_PROMETHEUS(self): + return self._get('ENABLE_PROMETHEUS', False) # type: bool + + @property + def PROMETHEUS_START_PORT(self): + # Workers will export metrics on this port, and ports following it + return self._get('PROMETHEUS_START_PORT', 9300) # type: int + + @property + def ATOMIC_JOBS(self): + return self._get('ATOMIC_JOBS', True) # type: bool + + +settings = Settings() diff --git a/django_lightweight_queue/backends/redis.py b/django_lightweight_queue/backends/redis.py index 6d4b689..72b5767 100644 --- a/django_lightweight_queue/backends/redis.py +++ b/django_lightweight_queue/backends/redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time +from ..app_settings import settings class RedisBackend(BackendWithPauseResume): @@ -17,9 +17,9 @@ class RedisBackend(BackendWithPauseResume): def __init__(self) -> None: self.client = redis.StrictRedis( - host=app_settings.REDIS_HOST, - port=app_settings.REDIS_PORT, - password=app_settings.REDIS_PASSWORD, + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, ) def enqueue(self, job: Job, queue: QueueName) -> None: @@ -79,9 +79,9 @@ def is_paused(self, queue: QueueName) -> bool: return bool(self.client.exists(self._pause_key(queue))) def _key(self, queue: QueueName) -> str: - if app_settings.REDIS_PREFIX: + if settings.REDIS_PREFIX: return '{}:django_lightweight_queue:{}'.format( - app_settings.REDIS_PREFIX, + settings.REDIS_PREFIX, queue, ) diff --git a/django_lightweight_queue/backends/reliable_redis.py b/django_lightweight_queue/backends/reliable_redis.py index 8012763..b01b5cc 100644 --- a/django_lightweight_queue/backends/reliable_redis.py +++ b/django_lightweight_queue/backends/reliable_redis.py @@ -3,11 +3,11 @@ import redis -from .. import app_settings from ..job import Job from .base import BackendWithDeduplicate, BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time, get_worker_numbers +from ..app_settings import settings from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER # Work around https://github.com/python/mypy/issues/9914. Name needs to match @@ -39,9 +39,9 @@ class ReliableRedisBackend(BackendWithDeduplicate, BackendWithPauseResume): def __init__(self) -> None: self.client = redis.StrictRedis( - host=app_settings.REDIS_HOST, - port=app_settings.REDIS_PORT, - password=app_settings.REDIS_PASSWORD, + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + password=settings.REDIS_PASSWORD, ) def startup(self, queue: QueueName) -> None: @@ -245,9 +245,9 @@ def _processing_key(self, queue: QueueName, worker_number: WorkerNumber) -> str: return self._prefix_key(key) def _prefix_key(self, key: str) -> str: - if app_settings.REDIS_PREFIX: + if settings.REDIS_PREFIX: return '{}:{}'.format( - app_settings.REDIS_PREFIX, + settings.REDIS_PREFIX, key, ) diff --git a/django_lightweight_queue/exposition.py b/django_lightweight_queue/exposition.py index d17f7ab..9ebba15 100644 --- a/django_lightweight_queue/exposition.py +++ b/django_lightweight_queue/exposition.py @@ -6,8 +6,8 @@ from prometheus_client.exposition import MetricsHandler -from . import app_settings from .types import QueueName, WorkerNumber +from .app_settings import settings def get_config_response( @@ -23,7 +23,7 @@ def get_config_response( "targets": [ "{}:{}".format( gethostname(), - app_settings.PROMETHEUS_START_PORT + index, + settings.PROMETHEUS_START_PORT + index, ), ], "labels": { @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): super(MetricsServer, self).__init__(*args, **kwargs) def run(self): - httpd = HTTPServer(('0.0.0.0', app_settings.PROMETHEUS_START_PORT), RequestHandler) + httpd = HTTPServer(('0.0.0.0', settings.PROMETHEUS_START_PORT), RequestHandler) httpd.timeout = 2 try: diff --git a/django_lightweight_queue/management/commands/queue_configuration.py b/django_lightweight_queue/management/commands/queue_configuration.py index feb6f4c..7c4aaf1 100644 --- a/django_lightweight_queue/management/commands/queue_configuration.py +++ b/django_lightweight_queue/management/commands/queue_configuration.py @@ -2,8 +2,8 @@ from django.core.management.base import BaseCommand, CommandParser -from ... import app_settings from ...utils import get_backend, get_queue_counts, load_extra_config +from ...app_settings import settings from ...cron_scheduler import get_cron_config @@ -37,7 +37,7 @@ def handle(self, **options: Any) -> None: print("") print("Middleware:") - for x in app_settings.MIDDLEWARE: + for x in settings.MIDDLEWARE: print(" * {}".format(x)) print("") diff --git a/django_lightweight_queue/runner.py b/django_lightweight_queue/runner.py index f3811e4..712c66f 100644 --- a/django_lightweight_queue/runner.py +++ b/django_lightweight_queue/runner.py @@ -4,10 +4,10 @@ import subprocess from typing import Dict, Tuple, Callable, Optional -from . import app_settings from .types import Logger, QueueName, WorkerNumber from .utils import get_backend, set_process_title from .exposition import metrics_http_server +from .app_settings import settings from .machine_types import Machine from .cron_scheduler import ( CronScheduler, @@ -64,7 +64,7 @@ def handle_term(signum: int, stack: object) -> None: for x in machine.worker_names } # type: Dict[Tuple[QueueName, WorkerNumber], Tuple[Optional[subprocess.Popen[bytes]], str]] - if app_settings.ENABLE_PROMETHEUS: + if settings.ENABLE_PROMETHEUS: metrics_server = metrics_http_server(machine.worker_names) metrics_server.start() @@ -107,7 +107,7 @@ def handle_term(signum: int, stack: object) -> None: queue, str(worker_num), '--prometheus-port', - str(app_settings.PROMETHEUS_START_PORT + index), + str(settings.PROMETHEUS_START_PORT + index), ] touch_filename = touch_filename_fn(queue) diff --git a/django_lightweight_queue/task.py b/django_lightweight_queue/task.py index 1e457c8..2da421f 100644 --- a/django_lightweight_queue/task.py +++ b/django_lightweight_queue/task.py @@ -12,10 +12,10 @@ Optional, ) -from . import app_settings from .job import Job from .types import QueueName from .utils import get_backend, contribute_implied_queue_name +from .app_settings import settings TCallable = TypeVar('TCallable', bound=Callable[..., Any]) @@ -82,7 +82,7 @@ def slow_fn(arg): """ if atomic is None: - atomic = app_settings.ATOMIC_JOBS + atomic = settings.ATOMIC_JOBS self.queue = QueueName(queue) self.timeout = timeout diff --git a/django_lightweight_queue/utils.py b/django_lightweight_queue/utils.py index 38a84f3..8299db1 100644 --- a/django_lightweight_queue/utils.py +++ b/django_lightweight_queue/utils.py @@ -21,8 +21,9 @@ from django.core.exceptions import MiddlewareNotUsed from django.utils.module_loading import module_has_submodule -from . import constants, app_settings +from . import constants from .types import Logger, QueueName, WorkerNumber +from .app_settings import settings if TYPE_CHECKING: from .backends.base import BaseBackend @@ -44,7 +45,7 @@ def with_prefix(names: Iterable[str]) -> Set[str]: for name in names ) - setting_names = get_setting_names(app_settings) + setting_names = get_setting_names(settings) extra_names = get_setting_names(extra_settings) unexpected_names = extra_names - with_prefix(setting_names) @@ -55,7 +56,7 @@ def with_prefix(names: Iterable[str]) -> Set[str]: override_names = extra_names - unexpected_names for name in override_names: short_name = name[len(constants.SETTING_NAME_PREFIX):] - setattr(app_settings, short_name, getattr(extra_settings, name)) + setattr(settings, short_name, getattr(extra_settings, name)) @lru_cache() @@ -69,19 +70,19 @@ def get_path(path: str) -> Any: @lru_cache() def get_backend(queue: QueueName) -> 'BaseBackend': - return get_path(app_settings.BACKEND_OVERRIDES.get( + return get_path(settings.BACKEND_OVERRIDES.get( queue, - app_settings.BACKEND, + settings.BACKEND, ))() @lru_cache() def get_logger(name: str) -> Logger: - get_logger_fn = app_settings.LOGGER_FACTORY + get_logger_fn = settings.LOGGER_FACTORY if not callable(get_logger_fn): get_logger_fn = cast( Callable[[str], Logger], - get_path(app_settings.LOGGER_FACTORY), + get_path(settings.LOGGER_FACTORY), ) return get_logger_fn(name) @@ -90,7 +91,7 @@ def get_logger(name: str) -> Logger: def get_middleware() -> List[Any]: middleware = [] - for path in app_settings.MIDDLEWARE: + for path in settings.MIDDLEWARE: try: middleware.append(get_path(path)()) except MiddlewareNotUsed: @@ -110,12 +111,12 @@ def contribute_implied_queue_name(queue: QueueName) -> None: "Queues have already been enumerated, ensure that " "'contribute_implied_queue_name' is called during setup.", ) - app_settings.WORKERS.setdefault(queue, 1) + settings.WORKERS.setdefault(queue, 1) def get_queue_counts() -> Mapping[QueueName, int]: refuse_further_implied_queues() - return app_settings.WORKERS + return settings.WORKERS def get_worker_numbers(queue: QueueName) -> Collection[WorkerNumber]: @@ -140,7 +141,7 @@ def import_all_submodules(name: str, exclude: Sequence[str] = ()) -> None: def load_all_tasks() -> None: - import_all_submodules('tasks', app_settings.IGNORE_APPS) + import_all_submodules('tasks', settings.IGNORE_APPS) def block_for_time( diff --git a/django_lightweight_queue/worker.py b/django_lightweight_queue/worker.py index da25dd2..63b9ee7 100644 --- a/django_lightweight_queue/worker.py +++ b/django_lightweight_queue/worker.py @@ -12,12 +12,12 @@ from django.db import connections, transaction -from . import app_settings from .types import QueueName, WorkerNumber from .utils import get_logger, get_backend, set_process_title +from .app_settings import settings from .backends.base import BaseBackend -if app_settings.ENABLE_PROMETHEUS: +if settings.ENABLE_PROMETHEUS: job_duration = Summary( 'item_processed_seconds', "Item processing time", @@ -54,7 +54,7 @@ def __init__( self.name = '{}/{}'.format(queue, worker_num) def run(self) -> None: - if app_settings.ENABLE_PROMETHEUS and self.prometheus_port is not None: + if settings.ENABLE_PROMETHEUS and self.prometheus_port is not None: self.log(logging.INFO, "Exporting metrics on port {}".format(self.prometheus_port)) start_http_server(self.prometheus_port) @@ -88,7 +88,7 @@ def run(self) -> None: item_processed = self.process(backend) post_process_time = time.time() - if app_settings.ENABLE_PROMETHEUS: + if settings.ENABLE_PROMETHEUS: job_duration.labels(self.queue).observe( post_process_time - pre_process_time, ) diff --git a/pyproject.toml b/pyproject.toml index 0c3996d..f78737d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-lightweight-queue" -version = "4.5.1" +version = "4.6.0" description = "Lightweight & modular queue and cron system for Django" authors = ["Thread Engineering "] license = "BSD-3-Clause" diff --git a/tests/test_pause_resume.py b/tests/test_pause_resume.py index eb70d38..842bb48 100644 --- a/tests/test_pause_resume.py +++ b/tests/test_pause_resume.py @@ -52,7 +52,7 @@ def setUp(self) -> None: # Can't use override_settings due to the copying of the settings values into # module values at startup. @mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', + 'django_lightweight_queue.app_settings.Settings.BACKEND', new='django_lightweight_queue.backends.redis.RedisBackend', ) def test_pause_resume(self) -> None: diff --git a/tests/test_reliable_redis_backend.py b/tests/test_reliable_redis_backend.py index 6359ca3..9d5ad71 100644 --- a/tests/test_reliable_redis_backend.py +++ b/tests/test_reliable_redis_backend.py @@ -49,8 +49,8 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: with unittest.mock.patch( 'django_lightweight_queue.utils._accepting_implied_queues', new=False, - ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', + ), unittest.mock.patch( + 'django_lightweight_queue.app_settings.Settings.WORKERS', workers, ): yield diff --git a/tests/test_task.py b/tests/test_task.py index 48f749b..0fe8a29 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -30,7 +30,7 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: 'django_lightweight_queue.utils._accepting_implied_queues', new=False, ), unittest.mock.patch.dict( - 'django_lightweight_queue.app_settings.WORKERS', + 'django_lightweight_queue.app_settings.Settings.WORKERS', workers, ): yield @@ -54,7 +54,7 @@ def mocked_get_path(path: str) -> Any: return get_path(path) patch = mock.patch( - 'django_lightweight_queue.app_settings.BACKEND', + 'django_lightweight_queue.app_settings.Settings.BACKEND', new='test-backend', ) patch.start() From c00635fd608f85a2f47baef03dd131bbfcacab64 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 18:44:04 -0400 Subject: [PATCH 02/14] make all settings changeable at runtime --- django_lightweight_queue/app_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 24ae5e0..15132a0 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -15,6 +15,7 @@ def _get(self, suffix: str, default: T) -> T: # adjustable values at runtime _backend = None + _redis_password = None @property def WORKERS(self): From 3b9e3a6044711fe619a44607eef84e1c2bbfb324 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 18:44:56 -0400 Subject: [PATCH 03/14] make all settings changeable at runtime, try 2 --- django_lightweight_queue/app_settings.py | 117 +++++++++++++++++++---- 1 file changed, 100 insertions(+), 17 deletions(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 15132a0..3cf40a5 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -14,12 +14,29 @@ def _get(self, suffix: str, default: T) -> T: return getattr(django_settings, attr_name, default) # adjustable values at runtime + _workers = None _backend = None + _logger_factory = None + _backend_overrides = None + _middleware = None + _ignore_apps = None + _redis_host = None + _redis_port = None _redis_password = None + _redis_prefix = None + _enable_prometheus = None + _prometheus_start_port = None + _atomic_jobs = None @property def WORKERS(self): - return self._get('WORKERS', {}) + if not self._workers: + self._workers = self._get('WORKERS', {}) + return self._workers + + @WORKERS.setter + def WORKERS(self, value): + self._workers = value @property def BACKEND(self): @@ -36,21 +53,39 @@ def BACKEND(self, value): @property def LOGGER_FACTORY(self): - return self._get( - 'LOGGER_FACTORY', - 'logging.getLogger', - ) # type: Union[str, Callable[[str], Logger]] + if not self._logger_factory: + self._logger_factory = self._get( + 'LOGGER_FACTORY', + 'logging.getLogger', + ) + return self._logger_factory # type: Union[str, Callable[[str], Logger]] + + @LOGGER_FACTORY.setter + def LOGGER_FACTORY(self, value): + self._logger_factory = value @property def BACKEND_OVERRIDES(self): # Allow per-queue overrides of the backend. - return self._get('BACKEND_OVERRIDES', {}) # type: Mapping[QueueName, str] + if not self._backend_overrides: + self._backend_overrides = self._get('BACKEND_OVERRIDES', {}) + return self._backend_overrides # type: Mapping[QueueName, str] + + @BACKEND_OVERRIDES.setter + def BACKEND_OVERRIDES(self, value): + self._backend_overrides = value @property def MIDDLEWARE(self): - return self._get('MIDDLEWARE', ( - 'django_lightweight_queue.middleware.logging.LoggingMiddleware', - )) # type: Sequence[str] + if not self._middleware: + self._middleware = self._get('MIDDLEWARE', ( + 'django_lightweight_queue.middleware.logging.LoggingMiddleware', + )) + return self._middleware # type: Sequence[str] + + @MIDDLEWARE.setter + def MIDDLEWARE(self, value): + self._middleware = value @property def IGNORE_APPS(self): @@ -59,36 +94,84 @@ def IGNORE_APPS(self): # have a file called `tasks.py` within an app, but don't want # django-lightweight-queue to import that file. # Note: this _doesn't_ prevent tasks being registered from these apps. - return self._get('IGNORE_APPS', ()) # type: Sequence[str] + if not self._ignore_apps: + self._ignore_apps = self._get('IGNORE_APPS', ()) + return self._ignore_apps # type: Sequence[str] + + @IGNORE_APPS.setter + def IGNORE_APPS(self, value): + self._ignore_apps = value @property def REDIS_HOST(self): - return self._get('REDIS_HOST', '127.0.0.1') # type: str + if not self._redis_host: + self._redis_host = self._get('REDIS_HOST', '127.0.0.1') + return self._redis_host # type: str + + @REDIS_HOST.setter + def REDIS_HOST(self, value): + self._redis_host = value @property def REDIS_PORT(self): - return self._get('REDIS_PORT', 6379) # type: int + if not self._redis_port: + self._redis_port = self._get('REDIS_PORT', 6379) + return self._redis_port # type: int + + @REDIS_PORT.setter + def REDIS_PORT(self, value): + self._redis_port = value @property def REDIS_PASSWORD(self): - return self._get('REDIS_PASSWORD', None) # type: Optional[str] + if not self._redis_password: + self._redis_password = self._get('REDIS_PASSWORD', None) + return self._redis_password # type: Optional[str] + + @REDIS_PASSWORD.setter + def REDIS_PASSWORD(self, value): + self._redis_password = value @property def REDIS_PREFIX(self): - return self._get('REDIS_PREFIX', '') # type: str + if not self._redis_prefix: + self._redis_prefix = self._get('REDIS_PREFIX', '') + return self._redis_prefix # type: str + + @REDIS_PREFIX.setter + def REDIS_PREFIX(self, value): + self._redis_prefix = value @property def ENABLE_PROMETHEUS(self): - return self._get('ENABLE_PROMETHEUS', False) # type: bool + if not self._enable_prometheus: + self._enable_prometheus = self._get('ENABLE_PROMETHEUS', False) + return self._enable_prometheus # type: bool + + @ENABLE_PROMETHEUS.setter + def ENABLE_PROMETHEUS(self, value): + self._enable_prometheus = value @property def PROMETHEUS_START_PORT(self): # Workers will export metrics on this port, and ports following it - return self._get('PROMETHEUS_START_PORT', 9300) # type: int + if not self._prometheus_start_port: + self._prometheus_start_port = self._get('PROMETHEUS_START_PORT', 9300) + return self._prometheus_start_port # type: int + + @PROMETHEUS_START_PORT.setter + def PROMETHEUS_START_PORT(self, value): + self._prometheus_start_port = value @property def ATOMIC_JOBS(self): - return self._get('ATOMIC_JOBS', True) # type: bool + if not self._atomic_jobs: + self._atomic_jobs = self._get('ATOMIC_JOBS', True) + return self._atomic_jobs # type: bool + + @ATOMIC_JOBS.setter + def ATOMIC_JOBS(self, value): + self._atomic_jobs = value settings = Settings() From 480a8a57136a11e873fbdc5160ac885062e3058f Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 19:39:16 -0400 Subject: [PATCH 04/14] fix reverse issue & add site_url setting --- django_lightweight_queue/app_settings.py | 13 +++++++++++++ django_lightweight_queue/backends/debug_web.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 3cf40a5..c120ed3 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -27,6 +27,7 @@ def _get(self, suffix: str, default: T) -> T: _enable_prometheus = None _prometheus_start_port = None _atomic_jobs = None + _site_url = None @property def WORKERS(self): @@ -173,5 +174,17 @@ def ATOMIC_JOBS(self): def ATOMIC_JOBS(self, value): self._atomic_jobs = value + @property + def SITE_URL(self): + if not self._site_url: + self._site_url = self._get('SITE_URL', True) + return self._site_url # type: bool + + @SITE_URL.setter + def SITE_URL(self, value): + self._site_url = value + + + settings = Settings() diff --git a/django_lightweight_queue/backends/debug_web.py b/django_lightweight_queue/backends/debug_web.py index b3f07de..08f89f5 100644 --- a/django_lightweight_queue/backends/debug_web.py +++ b/django_lightweight_queue/backends/debug_web.py @@ -20,7 +20,7 @@ class DebugWebBackend(BaseBackend): """ def enqueue(self, job: Job, queue: QueueName) -> None: - path = reverse('django-lightweight-queue:debug-run') + path = reverse('django_lightweight_queue:debug-run') query_string = urllib.parse.urlencode({'job': job.to_json()}) url = "{}{}?{}".format(settings.SITE_URL, path, query_string) print(url) From cfe8a502c529ff32af257ff7f50bed309df8dd95 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 19:43:01 -0400 Subject: [PATCH 05/14] pull from correct settings object --- django_lightweight_queue/app_settings.py | 2 -- django_lightweight_queue/backends/debug_web.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index c120ed3..11ff387 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -185,6 +185,4 @@ def SITE_URL(self, value): self._site_url = value - - settings = Settings() diff --git a/django_lightweight_queue/backends/debug_web.py b/django_lightweight_queue/backends/debug_web.py index 08f89f5..f0f4df3 100644 --- a/django_lightweight_queue/backends/debug_web.py +++ b/django_lightweight_queue/backends/debug_web.py @@ -1,11 +1,11 @@ import urllib.parse -from django.conf import settings from django.shortcuts import reverse from ..job import Job from .base import BaseBackend from ..types import QueueName, WorkerNumber +from ..app_settings import settings class DebugWebBackend(BaseBackend): From 62a71f850f8323d8250e8d906d971537764cfdb1 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 19:52:41 -0400 Subject: [PATCH 06/14] update documentation and add default for SITE_URL --- README.md | 36 ++++++++++++++++++++---- django_lightweight_queue/app_settings.py | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 68ecac1..6f77222 100644 --- a/README.md +++ b/README.md @@ -80,12 +80,36 @@ present in the specified file are inherited from the global configuration. There are four built-in backends: -| Backend | Import Location | Type | Description | -| -------------- |:----------------------------------------------------------------------| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Synchronous | django_lightweight_queue.backends.synchronous.SynchronousBackend | Development | Executes the task inline, without any actual queuing. | -| Redis | django_lightweight_queue.backends.redis.RedisBackend | Production | Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. | -| Reliable Redis | django_lightweight_queue.backends.reliable_redis.ReliableRedisBackend | Production | Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. | -| Debug Web | django_lightweight_queue.backends.debug_web.DebugWebBackend | Debugging | Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. | +### Synchronous (Development backend) + +`django_lightweight_queue.backends.synchronous.SynchronousBackend` + +Executes the task inline, without any actual queuing. + +### Redis (Production backend) + +`django_lightweight_queue.backends.redis.RedisBackend` + +Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. + +### Reliable Redis (Production backend) + +`django_lightweight_queue.backends.reliable_redis.ReliableRedisBackend` + +Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. + +### Debug Web (Debug backend) + +`django_lightweight_queue.backends.debug_web.DebugWebBackend` + +Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. + +This backend may require an extra setting if your debug site is not on localhost: + +```python +# defaults to http://localhost:8000 +LIGHTWEIGHT_QUEUE_SITE_URL = "http://example.com:8000" +``` [redis]: https://redis.io/ diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 11ff387..3cf0c8e 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -177,7 +177,7 @@ def ATOMIC_JOBS(self, value): @property def SITE_URL(self): if not self._site_url: - self._site_url = self._get('SITE_URL', True) + self._site_url = self._get('SITE_URL', "http://localhost:8000") return self._site_url # type: bool @SITE_URL.setter From aece0cc2485d1f4a87dcb6bb124ee0fd484000a9 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 18 Jun 2022 20:11:25 -0400 Subject: [PATCH 07/14] add missing configuration note about urls for debug web --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 6f77222..c90e6d4 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,21 @@ Executes tasks at-least-once using [Redis][redis] for storage of the enqueued ta Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. +Use this to append the appropriate URLs to the bottom of your root `urls.py`: + +```python +from django.conf import settings +from django.urls import path, include + +urlpatterns = [ + ... +] + +if "debug_web" in settings.LIGHTWEIGHT_QUEUE_BACKEND: + from django_lightweight_queue import urls as dlq_urls + urlpatterns += [path("", include(dlq_urls))] +``` + This backend may require an extra setting if your debug site is not on localhost: ```python From 737f80d47d531c879d21d6150cc27d5eb0b5f210 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Fri, 24 Jun 2022 19:17:32 -0400 Subject: [PATCH 08/14] fix missing type hint and revert name to app_settings --- django_lightweight_queue/app_settings.py | 8 +++---- django_lightweight_queue/backends/redis.py | 12 +++++----- .../backends/reliable_redis.py | 12 +++++----- django_lightweight_queue/exposition.py | 6 ++--- .../commands/queue_configuration.py | 4 ++-- django_lightweight_queue/runner.py | 6 ++--- django_lightweight_queue/task.py | 4 ++-- django_lightweight_queue/utils.py | 22 +++++++++---------- django_lightweight_queue/worker.py | 8 +++---- 9 files changed, 41 insertions(+), 41 deletions(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 3cf40a5..6b59e58 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,4 +1,4 @@ -from typing import Union, Mapping, TypeVar, Callable, Optional, Sequence +from typing import Union, Mapping, TypeVar, Callable, Optional, Sequence, Dict from django.conf import settings as django_settings @@ -8,7 +8,7 @@ T = TypeVar('T') -class Settings(): +class Settings: def _get(self, suffix: str, default: T) -> T: attr_name = '{}{}'.format(constants.SETTING_NAME_PREFIX, suffix) return getattr(django_settings, attr_name, default) @@ -32,7 +32,7 @@ def _get(self, suffix: str, default: T) -> T: def WORKERS(self): if not self._workers: self._workers = self._get('WORKERS', {}) - return self._workers + return self._workers # type: Dict[QueueName, int] @WORKERS.setter def WORKERS(self, value): @@ -174,4 +174,4 @@ def ATOMIC_JOBS(self, value): self._atomic_jobs = value -settings = Settings() +app_settings = Settings() diff --git a/django_lightweight_queue/backends/redis.py b/django_lightweight_queue/backends/redis.py index 72b5767..4a136f5 100644 --- a/django_lightweight_queue/backends/redis.py +++ b/django_lightweight_queue/backends/redis.py @@ -7,7 +7,7 @@ from .base import BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time -from ..app_settings import settings +from ..app_settings import app_settings class RedisBackend(BackendWithPauseResume): @@ -17,9 +17,9 @@ class RedisBackend(BackendWithPauseResume): def __init__(self) -> None: self.client = redis.StrictRedis( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, + host=app_settings.REDIS_HOST, + port=app_settings.REDIS_PORT, + password=app_settings.REDIS_PASSWORD, ) def enqueue(self, job: Job, queue: QueueName) -> None: @@ -79,9 +79,9 @@ def is_paused(self, queue: QueueName) -> bool: return bool(self.client.exists(self._pause_key(queue))) def _key(self, queue: QueueName) -> str: - if settings.REDIS_PREFIX: + if app_settings.REDIS_PREFIX: return '{}:django_lightweight_queue:{}'.format( - settings.REDIS_PREFIX, + app_settings.REDIS_PREFIX, queue, ) diff --git a/django_lightweight_queue/backends/reliable_redis.py b/django_lightweight_queue/backends/reliable_redis.py index b01b5cc..9eca0ad 100644 --- a/django_lightweight_queue/backends/reliable_redis.py +++ b/django_lightweight_queue/backends/reliable_redis.py @@ -7,7 +7,7 @@ from .base import BackendWithDeduplicate, BackendWithPauseResume from ..types import QueueName, WorkerNumber from ..utils import block_for_time, get_worker_numbers -from ..app_settings import settings +from ..app_settings import app_settings from ..progress_logger import ProgressLogger, NULL_PROGRESS_LOGGER # Work around https://github.com/python/mypy/issues/9914. Name needs to match @@ -39,9 +39,9 @@ class ReliableRedisBackend(BackendWithDeduplicate, BackendWithPauseResume): def __init__(self) -> None: self.client = redis.StrictRedis( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - password=settings.REDIS_PASSWORD, + host=app_settings.REDIS_HOST, + port=app_settings.REDIS_PORT, + password=app_settings.REDIS_PASSWORD, ) def startup(self, queue: QueueName) -> None: @@ -245,9 +245,9 @@ def _processing_key(self, queue: QueueName, worker_number: WorkerNumber) -> str: return self._prefix_key(key) def _prefix_key(self, key: str) -> str: - if settings.REDIS_PREFIX: + if app_settings.REDIS_PREFIX: return '{}:{}'.format( - settings.REDIS_PREFIX, + app_settings.REDIS_PREFIX, key, ) diff --git a/django_lightweight_queue/exposition.py b/django_lightweight_queue/exposition.py index 9ebba15..53fa53c 100644 --- a/django_lightweight_queue/exposition.py +++ b/django_lightweight_queue/exposition.py @@ -7,7 +7,7 @@ from prometheus_client.exposition import MetricsHandler from .types import QueueName, WorkerNumber -from .app_settings import settings +from .app_settings import app_settings def get_config_response( @@ -23,7 +23,7 @@ def get_config_response( "targets": [ "{}:{}".format( gethostname(), - settings.PROMETHEUS_START_PORT + index, + app_settings.PROMETHEUS_START_PORT + index, ), ], "labels": { @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): super(MetricsServer, self).__init__(*args, **kwargs) def run(self): - httpd = HTTPServer(('0.0.0.0', settings.PROMETHEUS_START_PORT), RequestHandler) + httpd = HTTPServer(('0.0.0.0', app_settings.PROMETHEUS_START_PORT), RequestHandler) httpd.timeout = 2 try: diff --git a/django_lightweight_queue/management/commands/queue_configuration.py b/django_lightweight_queue/management/commands/queue_configuration.py index 7c4aaf1..787ce98 100644 --- a/django_lightweight_queue/management/commands/queue_configuration.py +++ b/django_lightweight_queue/management/commands/queue_configuration.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand, CommandParser from ...utils import get_backend, get_queue_counts, load_extra_config -from ...app_settings import settings +from ...app_settings import app_settings from ...cron_scheduler import get_cron_config @@ -37,7 +37,7 @@ def handle(self, **options: Any) -> None: print("") print("Middleware:") - for x in settings.MIDDLEWARE: + for x in app_settings.MIDDLEWARE: print(" * {}".format(x)) print("") diff --git a/django_lightweight_queue/runner.py b/django_lightweight_queue/runner.py index 712c66f..11855b8 100644 --- a/django_lightweight_queue/runner.py +++ b/django_lightweight_queue/runner.py @@ -7,7 +7,7 @@ from .types import Logger, QueueName, WorkerNumber from .utils import get_backend, set_process_title from .exposition import metrics_http_server -from .app_settings import settings +from .app_settings import app_settings from .machine_types import Machine from .cron_scheduler import ( CronScheduler, @@ -64,7 +64,7 @@ def handle_term(signum: int, stack: object) -> None: for x in machine.worker_names } # type: Dict[Tuple[QueueName, WorkerNumber], Tuple[Optional[subprocess.Popen[bytes]], str]] - if settings.ENABLE_PROMETHEUS: + if app_settings.ENABLE_PROMETHEUS: metrics_server = metrics_http_server(machine.worker_names) metrics_server.start() @@ -107,7 +107,7 @@ def handle_term(signum: int, stack: object) -> None: queue, str(worker_num), '--prometheus-port', - str(settings.PROMETHEUS_START_PORT + index), + str(app_settings.PROMETHEUS_START_PORT + index), ] touch_filename = touch_filename_fn(queue) diff --git a/django_lightweight_queue/task.py b/django_lightweight_queue/task.py index 2da421f..25f57ad 100644 --- a/django_lightweight_queue/task.py +++ b/django_lightweight_queue/task.py @@ -15,7 +15,7 @@ from .job import Job from .types import QueueName from .utils import get_backend, contribute_implied_queue_name -from .app_settings import settings +from .app_settings import app_settings TCallable = TypeVar('TCallable', bound=Callable[..., Any]) @@ -82,7 +82,7 @@ def slow_fn(arg): """ if atomic is None: - atomic = settings.ATOMIC_JOBS + atomic = app_settings.ATOMIC_JOBS self.queue = QueueName(queue) self.timeout = timeout diff --git a/django_lightweight_queue/utils.py b/django_lightweight_queue/utils.py index 8299db1..cc51720 100644 --- a/django_lightweight_queue/utils.py +++ b/django_lightweight_queue/utils.py @@ -23,7 +23,7 @@ from . import constants from .types import Logger, QueueName, WorkerNumber -from .app_settings import settings +from .app_settings import app_settings if TYPE_CHECKING: from .backends.base import BaseBackend @@ -45,7 +45,7 @@ def with_prefix(names: Iterable[str]) -> Set[str]: for name in names ) - setting_names = get_setting_names(settings) + setting_names = get_setting_names(app_settings) extra_names = get_setting_names(extra_settings) unexpected_names = extra_names - with_prefix(setting_names) @@ -56,7 +56,7 @@ def with_prefix(names: Iterable[str]) -> Set[str]: override_names = extra_names - unexpected_names for name in override_names: short_name = name[len(constants.SETTING_NAME_PREFIX):] - setattr(settings, short_name, getattr(extra_settings, name)) + setattr(app_settings, short_name, getattr(extra_settings, name)) @lru_cache() @@ -70,19 +70,19 @@ def get_path(path: str) -> Any: @lru_cache() def get_backend(queue: QueueName) -> 'BaseBackend': - return get_path(settings.BACKEND_OVERRIDES.get( + return get_path(app_settings.BACKEND_OVERRIDES.get( queue, - settings.BACKEND, + app_settings.BACKEND, ))() @lru_cache() def get_logger(name: str) -> Logger: - get_logger_fn = settings.LOGGER_FACTORY + get_logger_fn = app_settings.LOGGER_FACTORY if not callable(get_logger_fn): get_logger_fn = cast( Callable[[str], Logger], - get_path(settings.LOGGER_FACTORY), + get_path(app_settings.LOGGER_FACTORY), ) return get_logger_fn(name) @@ -91,7 +91,7 @@ def get_logger(name: str) -> Logger: def get_middleware() -> List[Any]: middleware = [] - for path in settings.MIDDLEWARE: + for path in app_settings.MIDDLEWARE: try: middleware.append(get_path(path)()) except MiddlewareNotUsed: @@ -111,12 +111,12 @@ def contribute_implied_queue_name(queue: QueueName) -> None: "Queues have already been enumerated, ensure that " "'contribute_implied_queue_name' is called during setup.", ) - settings.WORKERS.setdefault(queue, 1) + app_settings.WORKERS.setdefault(queue, 1) def get_queue_counts() -> Mapping[QueueName, int]: refuse_further_implied_queues() - return settings.WORKERS + return app_settings.WORKERS def get_worker_numbers(queue: QueueName) -> Collection[WorkerNumber]: @@ -141,7 +141,7 @@ def import_all_submodules(name: str, exclude: Sequence[str] = ()) -> None: def load_all_tasks() -> None: - import_all_submodules('tasks', settings.IGNORE_APPS) + import_all_submodules('tasks', app_settings.IGNORE_APPS) def block_for_time( diff --git a/django_lightweight_queue/worker.py b/django_lightweight_queue/worker.py index 63b9ee7..565c31e 100644 --- a/django_lightweight_queue/worker.py +++ b/django_lightweight_queue/worker.py @@ -14,10 +14,10 @@ from .types import QueueName, WorkerNumber from .utils import get_logger, get_backend, set_process_title -from .app_settings import settings +from .app_settings import app_settings from .backends.base import BaseBackend -if settings.ENABLE_PROMETHEUS: +if app_settings.ENABLE_PROMETHEUS: job_duration = Summary( 'item_processed_seconds', "Item processing time", @@ -54,7 +54,7 @@ def __init__( self.name = '{}/{}'.format(queue, worker_num) def run(self) -> None: - if settings.ENABLE_PROMETHEUS and self.prometheus_port is not None: + if app_settings.ENABLE_PROMETHEUS and self.prometheus_port is not None: self.log(logging.INFO, "Exporting metrics on port {}".format(self.prometheus_port)) start_http_server(self.prometheus_port) @@ -88,7 +88,7 @@ def run(self) -> None: item_processed = self.process(backend) post_process_time = time.time() - if settings.ENABLE_PROMETHEUS: + if app_settings.ENABLE_PROMETHEUS: job_duration.labels(self.queue).observe( post_process_time - pre_process_time, ) From 0b202acce0a55c618c79cfd2ccd4a6378089abfb Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Fri, 24 Jun 2022 19:18:10 -0400 Subject: [PATCH 09/14] revert version number update --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f78737d..0c3996d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-lightweight-queue" -version = "4.6.0" +version = "4.5.1" description = "Lightweight & modular queue and cron system for Django" authors = ["Thread Engineering "] license = "BSD-3-Clause" From 97b36d47e4f201b0023d93341a0c3f69c51f28f8 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Fri, 24 Jun 2022 23:03:18 -0400 Subject: [PATCH 10/14] run isort --- django_lightweight_queue/app_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 6b59e58..29b43df 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,4 +1,4 @@ -from typing import Union, Mapping, TypeVar, Callable, Optional, Sequence, Dict +from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence from django.conf import settings as django_settings From 76631b4161029346c6ea47eff7a1f85ccc98373b Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Fri, 24 Jun 2022 23:10:27 -0400 Subject: [PATCH 11/14] move readme changes to own PR --- README.md | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 68ecac1..abce973 100644 --- a/README.md +++ b/README.md @@ -15,19 +15,6 @@ backends are great candidates for community contributions. ## Basic Usage -Start by adding `django_lightweight_queue` to your `INSTALLED_APPS`: - -```python -INSTALLED_APPS = [ - "django.contrib.admin", - "django.contrib.auth", - ..., - "django_lightweight_queue", -] -``` - -After that, define your task in any file you want: - ```python import time from django_lightweight_queue import task @@ -80,12 +67,12 @@ present in the specified file are inherited from the global configuration. There are four built-in backends: -| Backend | Import Location | Type | Description | -| -------------- |:----------------------------------------------------------------------| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Synchronous | django_lightweight_queue.backends.synchronous.SynchronousBackend | Development | Executes the task inline, without any actual queuing. | -| Redis | django_lightweight_queue.backends.redis.RedisBackend | Production | Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. | -| Reliable Redis | django_lightweight_queue.backends.reliable_redis.ReliableRedisBackend | Production | Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. | -| Debug Web | django_lightweight_queue.backends.debug_web.DebugWebBackend | Debugging | Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. | +| Backend | Type | Description | +| -------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Synchronous | Development | Executes the task inline, without any actual queuing. | +| Redis | Production | Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. | +| Reliable Redis | Production | Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. | +| Debug Web | Debugging | Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. | [redis]: https://redis.io/ From 73cfefe3ea3211fcd1fb476f47021307c79ba270 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 25 Jun 2022 00:09:25 -0400 Subject: [PATCH 12/14] appease mypy with appropriate blood-based rituals --- django_lightweight_queue/app_settings.py | 72 ++++++++++++++---------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index 29b43df..b23b89a 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -1,4 +1,4 @@ -from typing import Dict, Union, Mapping, TypeVar, Callable, Optional, Sequence +from typing import Any, Dict, Union, TypeVar, Callable, Optional, Sequence from django.conf import settings as django_settings @@ -28,67 +28,81 @@ def _get(self, suffix: str, default: T) -> T: _prometheus_start_port = None _atomic_jobs = None + def get_empty_dict(self) -> Dict[Any, Any]: + """ + Declare dummy type to make mypy happy. + + Mypy cannot handle types changing after instantiation, which is + exactly what happens to any setting that is a dict. This helper + method works around https://github.com/python/mypy/issues/6463 + and makes mypy happy -- at no point will we actually return + Dict[Any, Any] because between the time that mypy reads it and + the time that we actually need it, it will have been populated + with the value that we actually need it to be. + """ + return {} + @property - def WORKERS(self): + def WORKERS(self) -> Dict[QueueName, int]: if not self._workers: - self._workers = self._get('WORKERS', {}) - return self._workers # type: Dict[QueueName, int] + self._workers = self._get('WORKERS', self.get_empty_dict()) + return self._workers @WORKERS.setter def WORKERS(self, value): self._workers = value @property - def BACKEND(self): + def BACKEND(self) -> str: if not self._backend: self._backend = self._get( 'BACKEND', 'django_lightweight_queue.backends.synchronous.SynchronousBackend', ) - return self._backend # type: str + return self._backend @BACKEND.setter def BACKEND(self, value): self._backend = value @property - def LOGGER_FACTORY(self): + def LOGGER_FACTORY(self) -> Union[str, Callable[[str], Logger]]: if not self._logger_factory: self._logger_factory = self._get( 'LOGGER_FACTORY', 'logging.getLogger', ) - return self._logger_factory # type: Union[str, Callable[[str], Logger]] + return self._logger_factory @LOGGER_FACTORY.setter def LOGGER_FACTORY(self, value): self._logger_factory = value @property - def BACKEND_OVERRIDES(self): + def BACKEND_OVERRIDES(self) -> Dict[QueueName, str]: # Allow per-queue overrides of the backend. if not self._backend_overrides: - self._backend_overrides = self._get('BACKEND_OVERRIDES', {}) - return self._backend_overrides # type: Mapping[QueueName, str] + self._backend_overrides = self._get('BACKEND_OVERRIDES', self.get_empty_dict()) + return self._backend_overrides @BACKEND_OVERRIDES.setter def BACKEND_OVERRIDES(self, value): self._backend_overrides = value @property - def MIDDLEWARE(self): + def MIDDLEWARE(self) -> Sequence[str]: if not self._middleware: self._middleware = self._get('MIDDLEWARE', ( 'django_lightweight_queue.middleware.logging.LoggingMiddleware', )) - return self._middleware # type: Sequence[str] + return self._middleware @MIDDLEWARE.setter def MIDDLEWARE(self, value): self._middleware = value @property - def IGNORE_APPS(self): + def IGNORE_APPS(self) -> Sequence[str]: # Apps to ignore when looking for tasks. Apps must be specified as the dotted # name used in `INSTALLED_APPS`. This is expected to be useful when you need to # have a file called `tasks.py` within an app, but don't want @@ -96,78 +110,78 @@ def IGNORE_APPS(self): # Note: this _doesn't_ prevent tasks being registered from these apps. if not self._ignore_apps: self._ignore_apps = self._get('IGNORE_APPS', ()) - return self._ignore_apps # type: Sequence[str] + return self._ignore_apps @IGNORE_APPS.setter def IGNORE_APPS(self, value): self._ignore_apps = value @property - def REDIS_HOST(self): + def REDIS_HOST(self) -> str: if not self._redis_host: self._redis_host = self._get('REDIS_HOST', '127.0.0.1') - return self._redis_host # type: str + return self._redis_host @REDIS_HOST.setter def REDIS_HOST(self, value): self._redis_host = value @property - def REDIS_PORT(self): + def REDIS_PORT(self) -> int: if not self._redis_port: self._redis_port = self._get('REDIS_PORT', 6379) - return self._redis_port # type: int + return self._redis_port @REDIS_PORT.setter def REDIS_PORT(self, value): self._redis_port = value @property - def REDIS_PASSWORD(self): + def REDIS_PASSWORD(self) -> Optional[str]: if not self._redis_password: self._redis_password = self._get('REDIS_PASSWORD', None) - return self._redis_password # type: Optional[str] + return self._redis_password @REDIS_PASSWORD.setter def REDIS_PASSWORD(self, value): self._redis_password = value @property - def REDIS_PREFIX(self): + def REDIS_PREFIX(self) -> str: if not self._redis_prefix: self._redis_prefix = self._get('REDIS_PREFIX', '') - return self._redis_prefix # type: str + return self._redis_prefix @REDIS_PREFIX.setter def REDIS_PREFIX(self, value): self._redis_prefix = value @property - def ENABLE_PROMETHEUS(self): + def ENABLE_PROMETHEUS(self) -> bool: if not self._enable_prometheus: self._enable_prometheus = self._get('ENABLE_PROMETHEUS', False) - return self._enable_prometheus # type: bool + return self._enable_prometheus @ENABLE_PROMETHEUS.setter def ENABLE_PROMETHEUS(self, value): self._enable_prometheus = value @property - def PROMETHEUS_START_PORT(self): + def PROMETHEUS_START_PORT(self) -> int: # Workers will export metrics on this port, and ports following it if not self._prometheus_start_port: self._prometheus_start_port = self._get('PROMETHEUS_START_PORT', 9300) - return self._prometheus_start_port # type: int + return self._prometheus_start_port @PROMETHEUS_START_PORT.setter def PROMETHEUS_START_PORT(self, value): self._prometheus_start_port = value @property - def ATOMIC_JOBS(self): + def ATOMIC_JOBS(self) -> bool: if not self._atomic_jobs: self._atomic_jobs = self._get('ATOMIC_JOBS', True) - return self._atomic_jobs # type: bool + return self._atomic_jobs @ATOMIC_JOBS.setter def ATOMIC_JOBS(self, value): From 6ee7218123ec6aa03844dc6375c2a9e9e5969988 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 25 Jun 2022 00:21:05 -0400 Subject: [PATCH 13/14] revert mock to use .patch.dict() --- tests/test_reliable_redis_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_reliable_redis_backend.py b/tests/test_reliable_redis_backend.py index 9d5ad71..6a88210 100644 --- a/tests/test_reliable_redis_backend.py +++ b/tests/test_reliable_redis_backend.py @@ -49,8 +49,8 @@ def mock_workers(self, workers: Mapping[str, int]) -> Iterator[None]: with unittest.mock.patch( 'django_lightweight_queue.utils._accepting_implied_queues', new=False, - ), unittest.mock.patch( - 'django_lightweight_queue.app_settings.Settings.WORKERS', + ), unittest.mock.patch.dict( + 'django_lightweight_queue.app_settings.app_settings.WORKERS', workers, ): yield From 7bf8934547755521cc16fe9cbf5738f92aa4b437 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 25 Jun 2022 00:25:57 -0400 Subject: [PATCH 14/14] rename references and fix type hints --- django_lightweight_queue/app_settings.py | 4 ++-- django_lightweight_queue/backends/debug_web.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/django_lightweight_queue/app_settings.py b/django_lightweight_queue/app_settings.py index d20b219..ad33f34 100644 --- a/django_lightweight_queue/app_settings.py +++ b/django_lightweight_queue/app_settings.py @@ -189,10 +189,10 @@ def ATOMIC_JOBS(self, value): self._atomic_jobs = value @property - def SITE_URL(self): + def SITE_URL(self) -> str: if not self._site_url: self._site_url = self._get('SITE_URL', "http://localhost:8000") - return self._site_url # type: bool + return self._site_url @SITE_URL.setter def SITE_URL(self, value): diff --git a/django_lightweight_queue/backends/debug_web.py b/django_lightweight_queue/backends/debug_web.py index f0f4df3..8044805 100644 --- a/django_lightweight_queue/backends/debug_web.py +++ b/django_lightweight_queue/backends/debug_web.py @@ -5,7 +5,7 @@ from ..job import Job from .base import BaseBackend from ..types import QueueName, WorkerNumber -from ..app_settings import settings +from ..app_settings import app_settings class DebugWebBackend(BaseBackend): @@ -22,7 +22,7 @@ class DebugWebBackend(BaseBackend): def enqueue(self, job: Job, queue: QueueName) -> None: path = reverse('django_lightweight_queue:debug-run') query_string = urllib.parse.urlencode({'job': job.to_json()}) - url = "{}{}?{}".format(settings.SITE_URL, path, query_string) + url = "{}{}?{}".format(app_settings.SITE_URL, path, query_string) print(url) def dequeue(self, queue: QueueName, worker_num: WorkerNumber, timeout: float) -> None: