From 81f607f061a2ba2f0217863ff31059c7fb0c3e9d Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 1 Nov 2024 09:40:47 +0000 Subject: [PATCH] [AP-1794] bug fix for partial sync multiprocessing (#1186) * bug fix for partial sync multiprocessing * added method for setting start method as spawn * fix lint * handle lint complain * fix pep8 * revert set start method --- CHANGELOG.md | 5 +++++ pipelinewise/fastsync/partialsync/mysql_to_snowflake.py | 1 + pipelinewise/fastsync/partialsync/postgres_to_snowflake.py | 1 + setup.py | 2 +- 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5fd0a9b..dfd6be162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +0.66.1 (2024-10-31) +------------------- +- Bug fix for partial sync multiprocessing + + 0.66.0 (2024-10-11) ------------------- - Bump programming language to Python 3.10 diff --git a/pipelinewise/fastsync/partialsync/mysql_to_snowflake.py b/pipelinewise/fastsync/partialsync/mysql_to_snowflake.py index d974dbfde..0692341b2 100644 --- a/pipelinewise/fastsync/partialsync/mysql_to_snowflake.py +++ b/pipelinewise/fastsync/partialsync/mysql_to_snowflake.py @@ -138,6 +138,7 @@ def main_impl(): sync_tables = utils.get_sync_tables(args) + pool_size = len(sync_tables) if len(sync_tables) < pool_size else pool_size with multiprocessing.Pool(pool_size) as proc: sync_excs = list( filter( diff --git a/pipelinewise/fastsync/partialsync/postgres_to_snowflake.py b/pipelinewise/fastsync/partialsync/postgres_to_snowflake.py index 0f841c845..95e0afef7 100644 --- a/pipelinewise/fastsync/partialsync/postgres_to_snowflake.py +++ b/pipelinewise/fastsync/partialsync/postgres_to_snowflake.py @@ -135,6 +135,7 @@ def main_impl(): sync_tables = utils.get_sync_tables(args) + pool_size = len(sync_tables) if len(sync_tables) < pool_size else pool_size with multiprocessing.Pool(pool_size) as proc: sync_excs = list( filter( diff --git a/setup.py b/setup.py index e22b8cb4a..5c8b00446 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='pipelinewise', python_requires='==3.10.*', - version='0.66.0', + version='0.66.1', description='PipelineWise', long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown',