From 9f41c9b691fd98bab3b5566e16aa5379ceeacd9d Mon Sep 17 00:00:00 2001 From: Emmett McFaralne Date: Thu, 29 Aug 2024 12:11:06 -0400 Subject: [PATCH] set maxworkers on threads --- thepipe/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thepipe/scraper.py b/thepipe/scraper.py index 8de75c6..b33bdd9 100644 --- a/thepipe/scraper.py +++ b/thepipe/scraper.py @@ -201,7 +201,7 @@ def process_page(page_num): except Exception as e: raise ValueError(f"{e} (unable to read LLM response: {response})") - with concurrent.futures.ThreadPoolExecutor() as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: futures = [executor.submit(process_page, page_num) for page_num in range(num_pages)] page_results = OrderedDict() for future in concurrent.futures.as_completed(futures):