diff --git a/docs/user_guide/resync.rst b/docs/user_guide/resync.rst index 240bd0cc7..5ad86aee1 100644 --- a/docs/user_guide/resync.rst +++ b/docs/user_guide/resync.rst @@ -53,7 +53,7 @@ add the ``--tables`` argument: .. code-block:: yaml allowed_resync_max_size: - table_bytes: + table_mb: diff --git a/pipelinewise/fastsync/mysql_to_snowflake.py b/pipelinewise/fastsync/mysql_to_snowflake.py index 921efa649..5842d2e71 100644 --- a/pipelinewise/fastsync/mysql_to_snowflake.py +++ b/pipelinewise/fastsync/mysql_to_snowflake.py @@ -215,7 +215,7 @@ def main_impl(): all_tables_in_this_schema = get_tables_size(schema, tap_obj) only_selected_tables = filter_out_selected_tables(all_tables_in_this_schema, args.tables) table_with_maximum_size = get_maximum_value_from_list_of_dicts(only_selected_tables, 'table_size') - if table_with_maximum_size.get('table_size') > int(args.autoresync_size): + if table_with_maximum_size.get('table_size') > float(args.autoresync_size): can_run_sync = False table_sync_excs.append( f're-sync can not be done because size of table ' diff --git a/pipelinewise/fastsync/postgres_to_snowflake.py b/pipelinewise/fastsync/postgres_to_snowflake.py index 5b3cbd455..58ca4ecf1 100644 --- a/pipelinewise/fastsync/postgres_to_snowflake.py +++ b/pipelinewise/fastsync/postgres_to_snowflake.py @@ -213,7 +213,7 @@ def main_impl(): all_tables_in_this_schema = get_tables_size(schema, tap_obj) only_selected_tables = filter_out_selected_tables(all_tables_in_this_schema, args.tables) table_with_maximum_size = get_maximum_value_from_list_of_dicts(only_selected_tables, 'table_size') - if table_with_maximum_size.get('table_size') > int(args.autoresync_size): + if table_with_maximum_size.get('table_size') > float(args.autoresync_size): can_run_sync = False table_sync_excs.append( f're-sync can not be done because size of table ' diff --git a/pipelinewise/utils.py b/pipelinewise/utils.py index 642d6405d..9701fcf8c 100644 --- a/pipelinewise/utils.py +++ b/pipelinewise/utils.py @@ -31,7 +31,7 @@ def get_tables_size(schema: str, tap) -> dict: result_list = tap.query( 'select TABLE_NAME as table_name,' ' TABLE_ROWS as table_rows,' - ' ROUND((DATA_LENGTH + INDEX_LENGTH)/ 1024 / 1024) as table_size' + ' (DATA_LENGTH + INDEX_LENGTH)/ 1024 / 1024 as table_size' f' from information_schema.TABLES where TABLE_SCHEMA = \'{schema}\';') tap.close_connections() for res in result_list: @@ -48,7 +48,8 @@ def get_tables_size(schema: str, tap) -> dict: ' (xpath(\'/row/c/text()\',' ' query_to_xml(format(\'select count(*) as c from %I.%I\', table_schema, TABLE_NAME), FALSE, TRUE, \'\'))' ')[1]::text::int AS table_rows,' - ' pg_total_relation_size(\'"\'||table_schema||\'"."\'||table_name||\'"\')/1024/1024 as table_size ' + ' pg_total_relation_size(' + '\'"\'||table_schema||\'"."\'||table_name||\'"\')::NUMERIC/1024::NUMERIC/1024 as table_size ' 'FROM (SELECT table_schema, TABLE_NAME FROM information_schema.tables ' f'WHERE TABLE_NAME not like \'pg_%\' AND table_schema in (\'{schema}\')) as tb' ) diff --git a/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py index 7dd6f21b5..18c1600df 100644 --- a/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py +++ b/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py @@ -32,7 +32,7 @@ def tearDown(self): def test_resync_mariadb_to_sf_if_table_size_greater_than_limit(self): # pylint: disable = no-self-use """test resync mariadb to SF returns error 1 if table size is greater than the limit""" - a_small_number = 0.1 # Mb + a_small_number = 0.01 # Mb _create_ppw_config_file(table_mb=a_small_number) command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}' @@ -43,7 +43,7 @@ def test_resync_mariadb_to_sf_if_table_size_greater_than_limit(self): # pylint: def test_resync_mariadb_to_sf_if_table_size_less_than_limit(self): # pylint: disable = no-self-use """test resync mariadb to SF returns error if table size is less than the limit""" - a_big_number = 100000000000 + a_big_number = 10000 #Mb _create_ppw_config_file(table_mb=a_big_number) command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}' @@ -53,7 +53,7 @@ def test_resync_mariadb_to_sf_if_table_size_less_than_limit(self): # pylint: di def test_resync_mariadb_to_sf_if_table_size_greater_than_limit_and_force(self): # pylint: disable = no-self-use """test resync mariadb to SF returns error if table size is greater than the limit and --force is used""" - a_small_number = 0.1 # Mb + a_small_number = 0.01 # Mb _create_ppw_config_file(table_mb=a_small_number) command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID} --force' @@ -64,7 +64,7 @@ def test_resync_mariadb_to_sf_if_table_size_greater_than_limit_and_force(self): def test_run_tap_mariadb_to_sf_if_size_greater_than_limit(self): # pylint: disable = no-self-use """test run_tap mariadb to sf if table size is greater than the limit""" - a_small_number = 0.1 # Mb + a_small_number = 0.01 # Mb _create_ppw_config_file(table_mb=a_small_number) command = f'pipelinewise run_tap --tap {TAP_ID} --target {TARGET_ID}' diff --git a/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py index f5ae2af3a..6359b80ad 100644 --- a/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py +++ b/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py @@ -32,7 +32,7 @@ def tearDown(self): def test_resync_pg_to_sf_if_table_size_greater_than_limit(self): # pylint: disable = no-self-use """test resync pg to SF returns error 1 if table size is greater than the limit""" - a_small_number = 0.1 # Mb + a_small_number = 0.01 # Mb _create_ppw_config_file(table_mb=a_small_number) command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}' @@ -43,7 +43,7 @@ def test_resync_pg_to_sf_if_table_size_greater_than_limit(self): # pylint: dis def test_resync_pg_to_sf_if_table_size_less_than_limit(self): # pylint: disable = no-self-use """test resync pg to SF returns error if table size is less than the limit""" - a_big_number = 100000000000 + a_big_number = 1000 # Mb _create_ppw_config_file(table_mb=a_big_number) command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}' @@ -54,7 +54,7 @@ def test_resync_pg_to_sf_if_table_size_less_than_limit(self): # pylint: disabl def test_resync_pg_to_sf_if_table_size_greater_than_limit_and_force(self): # pylint: disable = no-self-use """test resync pg to SF returns error if table size is greater than the limit and --force is used""" - a_small_number = 0.1 # Mb + a_small_number = 0.01 # Mb _create_ppw_config_file(table_mb=a_small_number) command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID} --force' @@ -65,7 +65,7 @@ def test_resync_pg_to_sf_if_table_size_greater_than_limit_and_force(self): # p def test_run_tap_pg_to_sf_if_size_greater_than_limit(self): # pylint: disable = no-self-use """test run_tap postgres to sf if table size is greater than the limit""" - a_small_number = 0.1 # Mb + a_small_number = 0.01 # Mb _create_ppw_config_file(table_mb=a_small_number) command = f'pipelinewise run_tap --tap {TAP_ID} --target {TARGET_ID}'