diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py index 9faa88b54..4696c1330 100644 --- a/pipelinewise/cli/pipelinewise.py +++ b/pipelinewise/cli/pipelinewise.py @@ -1112,7 +1112,7 @@ def run_tap_fastsync( # Build the fastsync executable command max_autoresync_table_size = None if tap.type in ('tap-mysql', 'tap-postgres') and target.type == 'target-snowflake' and not self.force_fast_sync: - max_autoresync_table_size = self.config.get('allowed_resync_max_size', {}).get('table_bytes') + max_autoresync_table_size = self.config.get('allowed_resync_max_size', {}).get('table_mb') command = commands.build_fastsync_command( tap=tap, diff --git a/pipelinewise/fastsync/mysql_to_snowflake.py b/pipelinewise/fastsync/mysql_to_snowflake.py index fe952382c..b68cab8ce 100644 --- a/pipelinewise/fastsync/mysql_to_snowflake.py +++ b/pipelinewise/fastsync/mysql_to_snowflake.py @@ -219,7 +219,9 @@ def main_impl(): can_run_sync = False table_sync_excs.append( f're-sync can not be done because size of table ' - f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!') + f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!' + f' Use --force argument to force sync_tables!') + # Start loading tables in parallel in spawning processes if can_run_sync: diff --git a/pipelinewise/fastsync/postgres_to_snowflake.py b/pipelinewise/fastsync/postgres_to_snowflake.py index 8bcb81e10..5b3cbd455 100644 --- a/pipelinewise/fastsync/postgres_to_snowflake.py +++ b/pipelinewise/fastsync/postgres_to_snowflake.py @@ -217,7 +217,8 @@ def main_impl(): can_run_sync = False table_sync_excs.append( f're-sync can not be done because size of table ' - f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!') + f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!' + f' Use --force argument to force sync_tables!') # if internal arg drop_pg_slot is set to True, then we drop the slot before starting resync if args.drop_pg_slot: diff --git a/pipelinewise/utils.py b/pipelinewise/utils.py index becaf234c..642d6405d 100644 --- a/pipelinewise/utils.py +++ b/pipelinewise/utils.py @@ -31,7 +31,7 @@ def get_tables_size(schema: str, tap) -> dict: result_list = tap.query( 'select TABLE_NAME as table_name,' ' TABLE_ROWS as table_rows,' - ' DATA_LENGTH + INDEX_LENGTH as table_size' + ' ROUND((DATA_LENGTH + INDEX_LENGTH)/ 1024 / 1024) as table_size' f' from information_schema.TABLES where TABLE_SCHEMA = \'{schema}\';') tap.close_connections() for res in result_list: @@ -48,7 +48,7 @@ def get_tables_size(schema: str, tap) -> dict: ' (xpath(\'/row/c/text()\',' ' query_to_xml(format(\'select count(*) as c from %I.%I\', table_schema, TABLE_NAME), FALSE, TRUE, \'\'))' ')[1]::text::int AS table_rows,' - ' pg_total_relation_size(\'"\'||table_schema||\'"."\'||table_name||\'"\') as table_size ' + ' pg_total_relation_size(\'"\'||table_schema||\'"."\'||table_name||\'"\')/1024/1024 as table_size ' 'FROM (SELECT table_schema, TABLE_NAME FROM information_schema.tables ' f'WHERE TABLE_NAME not like \'pg_%\' AND table_schema in (\'{schema}\')) as tb' ) diff --git a/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py index d5d9ae75e..cf9d10e3b 100644 --- a/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py +++ b/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py @@ -9,10 +9,10 @@ TARGET_ID = 'snowflake' -def _create_ppw_config_file(table_byte): +def _create_ppw_config_file(table_mb): with open(f'{TEST_PROJECTS_DIR_PATH}/config.yml', 'w', encoding='utf-8') as config_file: config_file.write('allowed_resync_max_size:\n') - config_file.write(f' table_bytes: {table_byte}\n') + config_file.write(f' table_mb: {table_mb}\n') tasks.run_command(f'pipelinewise import_config --dir {TEST_PROJECTS_DIR_PATH}') diff --git a/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py index 34ee5a5fc..08dfd46c6 100644 --- a/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py +++ b/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py @@ -9,10 +9,10 @@ TARGET_ID = 'snowflake' -def _create_ppw_config_file(table_byte): +def _create_ppw_config_file(table_mb): with open(f'{TEST_PROJECTS_DIR_PATH}/config.yml', 'w', encoding='utf-8') as config_file: config_file.write('allowed_resync_max_size:\n') - config_file.write(f' table_bytes: {table_byte}\n') + config_file.write(f' table_mb: {table_mb}\n') tasks.run_command(f'pipelinewise import_config --dir {TEST_PROJECTS_DIR_PATH}')