[AP-1794] bug fix for partial sync multiprocessing (#1186)

* bug fix for partial sync multiprocessing * added method for setting start method as spawn * fix lint * handle lint complain * fix pep8 * revert set start method
transferwise · Nov 1, 2024 · 81f607f · 81f607f
1 parent 7f3192c
commit 81f607f
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,8 @@
+0.66.1 (2024-10-31)
+-------------------
+- Bug fix for partial sync multiprocessing
+
+
 0.66.0 (2024-10-11)
 -------------------
 - Bump programming language to Python 3.10

diff --git a/pipelinewise/fastsync/partialsync/mysql_to_snowflake.py b/pipelinewise/fastsync/partialsync/mysql_to_snowflake.py
@@ -138,6 +138,7 @@ def main_impl():
 
     sync_tables = utils.get_sync_tables(args)
 
+    pool_size = len(sync_tables) if len(sync_tables) < pool_size else pool_size
     with multiprocessing.Pool(pool_size) as proc:
         sync_excs = list(
             filter(

diff --git a/pipelinewise/fastsync/partialsync/postgres_to_snowflake.py b/pipelinewise/fastsync/partialsync/postgres_to_snowflake.py
@@ -135,6 +135,7 @@ def main_impl():
 
     sync_tables = utils.get_sync_tables(args)
 
+    pool_size = len(sync_tables) if len(sync_tables) < pool_size else pool_size
     with multiprocessing.Pool(pool_size) as proc:
         sync_excs = list(
             filter(

diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
 
 setup(name='pipelinewise',
       python_requires='==3.10.*',
-      version='0.66.0',
+      version='0.66.1',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',