Add OSIDB scheduler app and Stale Alert Collector with tests

collectors/osidb_scheduler/apps.py

@@ -0,0 +1,11 @@
+"""
+    OSIDB service
+"""
+
+from django.apps import AppConfig
+
+
+class OSIDBScheduler(AppConfig):
+    """OSIDB scheduler service"""
+
+    name = "collectors.osidb_scheduler"

collectors/osidb_scheduler/collectors.py

@@ -0,0 +1,67 @@
+"""
+OSIDB Collectors
+"""
+from celery.utils.log import get_task_logger
+from django.contrib.contenttypes.models import ContentType
+from django.db.models import OuterRef, Q, Subquery
+from django.db.models.functions import Cast
+
+from collectors.framework.models import Collector
+from osidb.mixins import Alert
+
+logger = get_task_logger(__name__)
+
+
+class StaleAlertCollector(Collector):
+    """
+    Stale Alert Collector
+    """
+
+    def collect(self):
+        """
+        collector run handler
+
+        On every run, this collector will check if the alert is
+        still valid by comparing the creation time of the alert
+        with the validation time of the Model.
+
+        If the creation time of the alert is older than
+        the validation time of the Model,
+        the alert is considered stale and will be deleted.
+        """
+        content_types = ContentType.objects.filter(
+            id__in=Alert.objects.values_list("content_type", flat=True).distinct()
+        )
+
+        logger.info(
+            f"Searching for stale alerts in {content_types.count()} content types"
+        )
+
+        query = Q()
+
+        for content_type in content_types:
+            model_class = content_type.model_class()
+
+            subquery = Subquery(
+                model_class.objects.filter(
+                    pk=Cast(OuterRef("object_id"), output_field=model_class._meta.pk)
+                ).values("last_validated_dt")[:1]
+            )
+
+            query |= Q(
+                content_type=content_type,
+                created_dt__lt=subquery,
+            )
+
+        filtered_queryset = Alert.objects.filter(query)
+
+        if filtered_queryset.count() == 0:
+            return "No Stale Alerts Found"
+
+        logger.info(f"Found {filtered_queryset.count()} stale alerts")
+
+        deleted_alerts_count = filtered_queryset.delete()[0]
+
+        logger.info(f"Deleted {deleted_alerts_count} stale alerts")
+
+        return f"Deleted {deleted_alerts_count} Stale Alerts"

collectors/osidb_scheduler/tasks.py

@@ -0,0 +1,23 @@
+"""
+Celery tasks for the OSIDB Collector
+"""
+
+from celery.schedules import crontab
+from celery.utils.log import get_task_logger
+
+from collectors.framework.models import collector
+from osidb.mixins import Alert
+
+from .collectors import StaleAlertCollector
+
+logger = get_task_logger(__name__)
+
+
+@collector(
+    base=StaleAlertCollector,
+    crontab=crontab(minute=0, hour="*/1"),  # Run every hour
+    data_models=[Alert],
+)
+def osidb_stale_alert_collector(collector_obj):
+    logger.info(f"Collector {collector_obj.name} is running")
+    return collector_obj.collect()

collectors/osidb_scheduler/tests/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+
+from collectors.osidb_scheduler.collectors import StaleAlertCollector
+
+
+@pytest.fixture(autouse=True)
+def use_debug(settings):
+    """Enforce DEBUG=True in all tests because pytest hardcodes it to False
+
+    See: https://github.com/pytest-dev/pytest-django/pull/463
+
+    Once the `--django-debug-mode` option is added to pytest, we can get rid of this fixture and
+    use the CLI setting via pytest.ini:
+    https://docs.pytest.org/en/latest/customize.html#adding-default-options
+    """
+    settings.DEBUG = True
+
+
+@pytest.fixture
+def stale_alert_collector():
+    return StaleAlertCollector()

collectors/osidb_scheduler/tests/test_collectors.py

@@ -0,0 +1,104 @@
+import pytest
+from django.utils import timezone
+from freezegun import freeze_time
+
+from osidb.mixins import Alert
+from osidb.models import Flaw, FlawSource
+from osidb.tests.factories import AffectFactory, FlawFactory, PsModuleFactory
+
+pytestmark = pytest.mark.unit
+
+
+class TestSlateAlertCollector:
+    @pytest.mark.vcr
+    def test_collect_flaw(self, stale_alert_collector):
+        """Test that the collector cleans up stale alerts related to a flaw."""
+        flaw = FlawFactory(
+            embargoed=False,
+            source=FlawSource.REDHAT,
+            major_incident_state=Flaw.FlawMajorIncident.MINOR,
+        )
+        flaw.save()
+        alerts = Alert.objects.filter(object_id=flaw.uuid)
+
+        assert alerts.count() == 3
+
+        with freeze_time(timezone.now() + timezone.timedelta(1)):
+            flaw.source = FlawSource.INTERNET
+            AffectFactory(flaw=flaw, ps_module=PsModuleFactory().name)
+            flaw.save()
+
+        alerts = Alert.objects.filter(object_id=flaw.uuid)
+        # Stale alerts still exist
+        assert alerts.count() == 3
+
+        result = stale_alert_collector.collect()
+        alerts = Alert.objects.filter(object_id=flaw.uuid)
+        # Stale alerts have been deleted
+        assert alerts.count() == 1
+        assert result == "Deleted 2 Stale Alerts"
+
+    @pytest.mark.vcr
+    def test_collect_affect(self, stale_alert_collector):
+        """Test that the collector cleans up stale alerts related to an affect."""
+        affect = AffectFactory()
+        alerts = Alert.objects.filter(object_id=affect.uuid)
+
+        assert alerts.count() == 1
+
+        with freeze_time(timezone.now() + timezone.timedelta(1)):
+            affect.ps_module = PsModuleFactory().name
+            affect.save()
+
+        alerts = Alert.objects.filter(object_id=affect.uuid)
+        # Stale alerts still exist
+        assert alerts.count() == 1
+
+        result = stale_alert_collector.collect()
+        alerts = Alert.objects.filter(object_id=affect.uuid)
+        # Stale alerts have been deleted
+        assert alerts.count() == 0
+        assert result == "Deleted 1 Stale Alerts"
+
+    @pytest.mark.vcr
+    def test_collect_multi(self, stale_alert_collector):
+        """Test that the collector cleans up stale alerts related to multiple objects."""
+        flaw = FlawFactory(
+            embargoed=False,
+            source=FlawSource.REDHAT,
+            major_incident_state=Flaw.FlawMajorIncident.MINOR,
+        )
+        flaw.save()
+        alerts = Alert.objects.filter(object_id=flaw.uuid)
+
+        assert alerts.count() == 3
+
+        affect = AffectFactory(flaw=flaw)
+        alerts = Alert.objects.filter(object_id=affect.uuid)
+
+        assert alerts.count() == 1
+
+        with freeze_time(timezone.now() + timezone.timedelta(1)):
+            flaw.source = FlawSource.INTERNET
+            affect.ps_module = PsModuleFactory().name
+            flaw.save()
+            affect.save()
+
+        alerts = Alert.objects.filter(object_id=flaw.uuid)
+        # Stale alerts still exist
+        assert alerts.count() == 3
+
+        alerts = Alert.objects.filter(object_id=affect.uuid)
+        # Stale alerts still exist
+        assert alerts.count() == 1
+
+        result = stale_alert_collector.collect()
+        alerts = Alert.objects.filter(object_id=flaw.uuid)
+        # Stale alerts have been deleted
+        assert alerts.count() == 1
+
+        alerts = Alert.objects.filter(object_id=affect.uuid)
+        # Stale alerts have been deleted
+        assert alerts.count() == 0
+
+        assert result == "Deleted 3 Stale Alerts"

config/settings.py

@@ -51,6 +51,7 @@
     "collectors.jiraffe",
     "collectors.nvd",
     "collectors.osv",
+    "collectors.osidb_scheduler",
     "collectors.product_definitions",
     "collectors.ps_constants",
     "corsheaders",
@@ -244,6 +245,7 @@
                 "collectors.jiraffe",
                 "collectors.nvd",
                 "collectors.osv",
+                "collectors.osidb_scheduler",
                 "collectors.product_definitions",
                 "collectors.ps_constants",
             ]