Change batch size to 200 and enable multiprocessing

Signed-off-by: Tushar Goel <[email protected]>
aboutcode-org · Feb 26, 2025 · e36500b · e36500b
1 parent b1b40f5
commit e36500b
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/vulnerabilities/pipelines/recompute_content_ids.py b/vulnerabilities/pipelines/recompute_content_ids.py
@@ -94,8 +94,7 @@ def process_advisories(
     )
 
     log(f"Running function: {advisory_func.__name__}", level=logging.INFO)
-    # if max_workers <= 0:
-    if True:
+    if max_workers <= 0:
         log(f"Running function in single process", level=logging.INFO)
         for advisory_ids in progress.iter(advisory_batches):
             progress.log_progress()

diff --git a/vulnerabilities/pipelines/remove_duplicate_advisories.py b/vulnerabilities/pipelines/remove_duplicate_advisories.py
@@ -73,7 +73,7 @@ class RemoveDuplicateAdvisoriesPipeline(VulnerableCodePipeline):
     """Pipeline to remove duplicate advisories based on their content."""
 
     pipeline_id = "remove_duplicate_advisories"
-    BATCH_SIZE = 1000
+    BATCH_SIZE = 200
 
     @classmethod
     def steps(cls):