Collect keywords from ps-constants

collectors/cveorg/migrations/0001_initial.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.16 on 2024-12-10 14:05
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Keyword',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('keyword', models.CharField(max_length=255, unique=True)),
+                ('type', models.CharField(choices=[('ALLOWLIST', 'Allowlist'), ('ALLOWLIST_SPECIAL_CASE', 'Allowlist Special Case'), ('BLOCKLIST', 'Blocklist'), ('BLOCKLIST_SPECIAL_CASE', 'Blocklist Special Case')], max_length=25)),
+            ],
+        ),
+    ]

collectors/cveorg/models.py

@@ -0,0 +1,23 @@
+import uuid
+
+from django.db import models
+
+
+class Keyword(models.Model):
+    """
+    An instance of this model represents a keyword of a given type
+    collected from `data/cveorg_keywords.yml` in the `ps-constants` repository.
+
+    These keywords determine whether the CVEorg collector should create a flaw.
+    """
+
+    class Type(models.TextChoices):
+        ALLOWLIST = "ALLOWLIST"
+        ALLOWLIST_SPECIAL_CASE = "ALLOWLIST_SPECIAL_CASE"
+        BLOCKLIST = "BLOCKLIST"
+        BLOCKLIST_SPECIAL_CASE = "BLOCKLIST_SPECIAL_CASE"
+
+    # internal primary key
+    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    keyword = models.CharField(max_length=255, unique=True)
+    type = models.CharField(choices=Type.choices, max_length=25)

collectors/ps_constants/core.py

@@ -8,6 +8,7 @@
 
 from apps.sla.models import SLA, SLAPolicy
 from apps.trackers.models import JiraBugIssuetype
+from collectors.cveorg.models import Keyword
 from osidb.models import SpecialConsiderationPackage
 
 logger = logging.getLogger(__name__)
@@ -68,3 +69,28 @@ def sync_jira_bug_issuetype(source_dict):
     JiraBugIssuetype.objects.all().delete()
     for project in list(source_dict.values())[0]:
         JiraBugIssuetype.objects.get_or_create(project=project)
+
+
+@transaction.atomic
+def sync_cveorg_keywords(source: dict) -> None:
+    """
+    Sync CVEorg keywords in the database
+    """
+    try:
+        keywords = [
+            (Keyword.Type.ALLOWLIST, source["allowlist"]),
+            (Keyword.Type.ALLOWLIST_SPECIAL_CASE, source["allowlist_special_cases"]),
+            (Keyword.Type.BLOCKLIST, source["blocklist"]),
+            (Keyword.Type.BLOCKLIST_SPECIAL_CASE, source["blocklist_special_cases"]),
+        ]
+    except KeyError:
+        raise KeyError(
+            "The ps-constants repository does not contain the expected CVEorg keyword sections."
+        )
+
+    # Delete and recreate keywords
+    Keyword.objects.all().delete()
+    for keyword_type, data in keywords:
+        for entry in data:
+            keyword = Keyword(keyword=entry, type=keyword_type)
+            keyword.save()

collectors/ps_constants/tasks.py

@@ -12,6 +12,7 @@
 from .constants import PS_CONSTANTS_REPO_BRANCH, PS_CONSTANTS_REPO_URL
 from .core import (
     fetch_ps_constants,
+    sync_cveorg_keywords,
     sync_jira_bug_issuetype,
     sync_sla_policies,
     sync_special_consideration_packages,
@@ -46,18 +47,25 @@ def collect_step_1_fetch():
     logger.info(f"Fetching PS Constants (Jira Bug issuetype) from '{url}'")
     jira_bug_issuetype = fetch_ps_constants(url)
 
+    url = f"{PS_CONSTANTS_BASE_URL}/cveorg_keywords.yml"
+    logger.info(f"Fetching CVEorg keywords from '{url}'")
+    cveorg_keywords = fetch_ps_constants(url)
+
     return (
+        cveorg_keywords,
         sc_packages,
         sla_policies,
         jira_bug_issuetype,
     )
 
 
 def collect_step_2_sync(
+    cveorg_keywords,
     sc_packages,
     sla_policies,
     jira_bug_issuetype,
 ):
+    sync_cveorg_keywords(cveorg_keywords)
     sync_special_consideration_packages(sc_packages)
     sync_sla_policies(sla_policies)
     sync_jira_bug_issuetype(jira_bug_issuetype)
@@ -83,6 +91,7 @@ def ps_constants_collector(collector_obj) -> str:
     """ps constants collector"""
 
     (
+        cveorg_keywords,
         sc_packages,
         sla_policies,
         jira_bug_issuetype,
@@ -96,6 +105,7 @@ def ps_constants_collector(collector_obj) -> str:
     )
 
     collect_step_2_sync(
+        cveorg_keywords,
         sc_packages,
         sla_policies,
         jira_bug_issuetype,