From d979b0eb95810829cf8801c28542c11f52770986 Mon Sep 17 00:00:00 2001 From: Alvaro Tinoco Date: Tue, 10 Dec 2024 13:40:02 +0100 Subject: [PATCH] Add OSIDB scheduler app and Stale Alert Collector with tests --- collectors/osidb_scheduler/__init__.py | 0 collectors/osidb_scheduler/apps.py | 11 ++ collectors/osidb_scheduler/collectors.py | 67 +++++++++++ collectors/osidb_scheduler/tasks.py | 23 ++++ collectors/osidb_scheduler/tests/__init__.py | 0 collectors/osidb_scheduler/tests/conftest.py | 21 ++++ .../osidb_scheduler/tests/test_collectors.py | 104 ++++++++++++++++++ config/settings.py | 2 + 8 files changed, 228 insertions(+) create mode 100644 collectors/osidb_scheduler/__init__.py create mode 100644 collectors/osidb_scheduler/apps.py create mode 100644 collectors/osidb_scheduler/collectors.py create mode 100644 collectors/osidb_scheduler/tasks.py create mode 100644 collectors/osidb_scheduler/tests/__init__.py create mode 100644 collectors/osidb_scheduler/tests/conftest.py create mode 100644 collectors/osidb_scheduler/tests/test_collectors.py diff --git a/collectors/osidb_scheduler/__init__.py b/collectors/osidb_scheduler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/collectors/osidb_scheduler/apps.py b/collectors/osidb_scheduler/apps.py new file mode 100644 index 000000000..3dcb4ed82 --- /dev/null +++ b/collectors/osidb_scheduler/apps.py @@ -0,0 +1,11 @@ +""" + OSIDB service +""" + +from django.apps import AppConfig + + +class OSIDBScheduler(AppConfig): + """OSIDB scheduler service""" + + name = "collectors.osidb_scheduler" diff --git a/collectors/osidb_scheduler/collectors.py b/collectors/osidb_scheduler/collectors.py new file mode 100644 index 000000000..c11e7405d --- /dev/null +++ b/collectors/osidb_scheduler/collectors.py @@ -0,0 +1,67 @@ +""" +OSIDB Collectors +""" +from celery.utils.log import get_task_logger +from django.contrib.contenttypes.models import ContentType +from django.db.models import OuterRef, Q, Subquery +from django.db.models.functions import Cast + +from collectors.framework.models import Collector +from osidb.mixins import Alert + +logger = get_task_logger(__name__) + + +class StaleAlertCollector(Collector): + """ + Stale Alert Collector + """ + + def collect(self): + """ + collector run handler + + On every run, this collector will check if the alert is + still valid by comparing the creation time of the alert + with the validation time of the Model. + + If the creation time of the alert is older than + the validation time of the Model, + the alert is considered stale and will be deleted. + """ + content_types = ContentType.objects.filter( + id__in=Alert.objects.values_list("content_type", flat=True).distinct() + ) + + logger.info( + f"Searching for stale alerts in {content_types.count()} content types" + ) + + query = Q() + + for content_type in content_types: + model_class = content_type.model_class() + + subquery = Subquery( + model_class.objects.filter( + pk=Cast(OuterRef("object_id"), output_field=model_class._meta.pk) + ).values("last_validated_dt")[:1] + ) + + query |= Q( + content_type=content_type, + created_dt__lt=subquery, + ) + + filtered_queryset = Alert.objects.filter(query) + + if filtered_queryset.count() == 0: + return "No Stale Alerts Found" + + logger.info(f"Found {filtered_queryset.count()} stale alerts") + + deleted_alerts_count = filtered_queryset.delete()[0] + + logger.info(f"Deleted {deleted_alerts_count} stale alerts") + + return f"Deleted {deleted_alerts_count} Stale Alerts" diff --git a/collectors/osidb_scheduler/tasks.py b/collectors/osidb_scheduler/tasks.py new file mode 100644 index 000000000..33c428d9e --- /dev/null +++ b/collectors/osidb_scheduler/tasks.py @@ -0,0 +1,23 @@ +""" +Celery tasks for the OSIDB Collector +""" + +from celery.schedules import crontab +from celery.utils.log import get_task_logger + +from collectors.framework.models import collector +from osidb.mixins import Alert + +from .collectors import StaleAlertCollector + +logger = get_task_logger(__name__) + + +@collector( + base=StaleAlertCollector, + crontab=crontab(minute=0, hour="*/1"), # Run every hour + data_models=[Alert], +) +def osidb_stale_alert_collector(collector_obj): + logger.info(f"Collector {collector_obj.name} is running") + return collector_obj.collect() diff --git a/collectors/osidb_scheduler/tests/__init__.py b/collectors/osidb_scheduler/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/collectors/osidb_scheduler/tests/conftest.py b/collectors/osidb_scheduler/tests/conftest.py new file mode 100644 index 000000000..f1ca2a88c --- /dev/null +++ b/collectors/osidb_scheduler/tests/conftest.py @@ -0,0 +1,21 @@ +import pytest + +from collectors.osidb_scheduler.collectors import StaleAlertCollector + + +@pytest.fixture(autouse=True) +def use_debug(settings): + """Enforce DEBUG=True in all tests because pytest hardcodes it to False + + See: https://github.com/pytest-dev/pytest-django/pull/463 + + Once the `--django-debug-mode` option is added to pytest, we can get rid of this fixture and + use the CLI setting via pytest.ini: + https://docs.pytest.org/en/latest/customize.html#adding-default-options + """ + settings.DEBUG = True + + +@pytest.fixture +def stale_alert_collector(): + return StaleAlertCollector() diff --git a/collectors/osidb_scheduler/tests/test_collectors.py b/collectors/osidb_scheduler/tests/test_collectors.py new file mode 100644 index 000000000..791c7ca32 --- /dev/null +++ b/collectors/osidb_scheduler/tests/test_collectors.py @@ -0,0 +1,104 @@ +import pytest +from django.utils import timezone +from freezegun import freeze_time + +from osidb.mixins import Alert +from osidb.models import Flaw, FlawSource +from osidb.tests.factories import AffectFactory, FlawFactory, PsModuleFactory + +pytestmark = pytest.mark.unit + + +class TestSlateAlertCollector: + @pytest.mark.vcr + def test_collect_flaw(self, stale_alert_collector): + """Test that the collector cleans up stale alerts related to a flaw.""" + flaw = FlawFactory( + embargoed=False, + source=FlawSource.REDHAT, + major_incident_state=Flaw.FlawMajorIncident.MINOR, + ) + flaw.save() + alerts = Alert.objects.filter(object_id=flaw.uuid) + + assert alerts.count() == 3 + + with freeze_time(timezone.now() + timezone.timedelta(1)): + flaw.source = FlawSource.INTERNET + AffectFactory(flaw=flaw, ps_module=PsModuleFactory().name) + flaw.save() + + alerts = Alert.objects.filter(object_id=flaw.uuid) + # Stale alerts still exist + assert alerts.count() == 3 + + result = stale_alert_collector.collect() + alerts = Alert.objects.filter(object_id=flaw.uuid) + # Stale alerts have been deleted + assert alerts.count() == 1 + assert result == "Deleted 2 Stale Alerts" + + @pytest.mark.vcr + def test_collect_affect(self, stale_alert_collector): + """Test that the collector cleans up stale alerts related to an affect.""" + affect = AffectFactory() + alerts = Alert.objects.filter(object_id=affect.uuid) + + assert alerts.count() == 1 + + with freeze_time(timezone.now() + timezone.timedelta(1)): + affect.ps_module = PsModuleFactory().name + affect.save() + + alerts = Alert.objects.filter(object_id=affect.uuid) + # Stale alerts still exist + assert alerts.count() == 1 + + result = stale_alert_collector.collect() + alerts = Alert.objects.filter(object_id=affect.uuid) + # Stale alerts have been deleted + assert alerts.count() == 0 + assert result == "Deleted 1 Stale Alerts" + + @pytest.mark.vcr + def test_collect_multi(self, stale_alert_collector): + """Test that the collector cleans up stale alerts related to multiple objects.""" + flaw = FlawFactory( + embargoed=False, + source=FlawSource.REDHAT, + major_incident_state=Flaw.FlawMajorIncident.MINOR, + ) + flaw.save() + alerts = Alert.objects.filter(object_id=flaw.uuid) + + assert alerts.count() == 3 + + affect = AffectFactory(flaw=flaw) + alerts = Alert.objects.filter(object_id=affect.uuid) + + assert alerts.count() == 1 + + with freeze_time(timezone.now() + timezone.timedelta(1)): + flaw.source = FlawSource.INTERNET + affect.ps_module = PsModuleFactory().name + flaw.save() + affect.save() + + alerts = Alert.objects.filter(object_id=flaw.uuid) + # Stale alerts still exist + assert alerts.count() == 3 + + alerts = Alert.objects.filter(object_id=affect.uuid) + # Stale alerts still exist + assert alerts.count() == 1 + + result = stale_alert_collector.collect() + alerts = Alert.objects.filter(object_id=flaw.uuid) + # Stale alerts have been deleted + assert alerts.count() == 1 + + alerts = Alert.objects.filter(object_id=affect.uuid) + # Stale alerts have been deleted + assert alerts.count() == 0 + + assert result == "Deleted 3 Stale Alerts" diff --git a/config/settings.py b/config/settings.py index 4de8c7048..2798e3b14 100644 --- a/config/settings.py +++ b/config/settings.py @@ -51,6 +51,7 @@ "collectors.jiraffe", "collectors.nvd", "collectors.osv", + "collectors.osidb_scheduler", "collectors.product_definitions", "collectors.ps_constants", "corsheaders", @@ -244,6 +245,7 @@ "collectors.jiraffe", "collectors.nvd", "collectors.osv", + "collectors.osidb_scheduler", "collectors.product_definitions", "collectors.ps_constants", ]