set maxworkers on threads

emcf · Aug 29, 2024 · 9f41c9b · 9f41c9b
1 parent afc35e5
commit 9f41c9b
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/thepipe/scraper.py b/thepipe/scraper.py
@@ -201,7 +201,7 @@ def process_page(page_num):
                 except Exception as e:
                     raise ValueError(f"{e} (unable to read LLM response: {response})")
 
-            with concurrent.futures.ThreadPoolExecutor() as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
                 futures = [executor.submit(process_page, page_num) for page_num in range(num_pages)]
                 page_results = OrderedDict()
                 for future in concurrent.futures.as_completed(futures):