changed setting to MB

transferwise · Jul 18, 2024 · 6b28704 · 6b28704
1 parent 108effb
commit 6b28704
Show file tree

Hide file tree

Showing 6 changed files with 12 additions and 9 deletions.
diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
@@ -1112,7 +1112,7 @@ def run_tap_fastsync(
         # Build the fastsync executable command
         max_autoresync_table_size = None
         if tap.type in ('tap-mysql', 'tap-postgres') and target.type == 'target-snowflake' and not self.force_fast_sync:
-            max_autoresync_table_size = self.config.get('allowed_resync_max_size', {}).get('table_bytes')
+            max_autoresync_table_size = self.config.get('allowed_resync_max_size', {}).get('table_mb')
 
         command = commands.build_fastsync_command(
             tap=tap,

diff --git a/pipelinewise/fastsync/mysql_to_snowflake.py b/pipelinewise/fastsync/mysql_to_snowflake.py
@@ -219,7 +219,9 @@ def main_impl():
                 can_run_sync = False
                 table_sync_excs.append(
                     f're-sync can not be done because size of table '
-                    f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!')
+                    f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!'
+                    f' Use --force argument to force sync_tables!')
+
 
     # Start loading tables in parallel in spawning processes
     if can_run_sync:

diff --git a/pipelinewise/fastsync/postgres_to_snowflake.py b/pipelinewise/fastsync/postgres_to_snowflake.py
@@ -217,7 +217,8 @@ def main_impl():
                 can_run_sync = False
                 table_sync_excs.append(
                     f're-sync can not be done because size of table '
-                    f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!')
+                    f'`{table_with_maximum_size["table_name"]}` is greater than `{args.autoresync_size}`!'
+                    f' Use --force argument to force sync_tables!')
 
     # if internal arg drop_pg_slot is set to True, then we drop the slot before starting resync
     if args.drop_pg_slot:

diff --git a/pipelinewise/utils.py b/pipelinewise/utils.py
@@ -31,7 +31,7 @@ def get_tables_size(schema: str, tap) -> dict:
         result_list = tap.query(
             'select TABLE_NAME as table_name,'
             ' TABLE_ROWS as table_rows,'
-            ' DATA_LENGTH + INDEX_LENGTH as table_size'
+            ' ROUND((DATA_LENGTH + INDEX_LENGTH)/ 1024 / 1024) as table_size'
             f' from information_schema.TABLES where TABLE_SCHEMA = \'{schema}\';')
         tap.close_connections()
         for res in result_list:
@@ -48,7 +48,7 @@ def get_tables_size(schema: str, tap) -> dict:
             ' (xpath(\'/row/c/text()\','
             ' query_to_xml(format(\'select count(*) as c from %I.%I\', table_schema, TABLE_NAME), FALSE, TRUE, \'\'))'
             ')[1]::text::int AS table_rows,'
-            ' pg_total_relation_size(\'"\'||table_schema||\'"."\'||table_name||\'"\') as table_size '
+            ' pg_total_relation_size(\'"\'||table_schema||\'"."\'||table_name||\'"\')/1024/1024 as table_size '
             'FROM (SELECT table_schema, TABLE_NAME FROM information_schema.tables '
             f'WHERE TABLE_NAME not like \'pg_%\' AND table_schema in (\'{schema}\')) as tb'
             )

diff --git a/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py
@@ -9,10 +9,10 @@
 TARGET_ID = 'snowflake'
 
 
-def _create_ppw_config_file(table_byte):
+def _create_ppw_config_file(table_mb):
     with open(f'{TEST_PROJECTS_DIR_PATH}/config.yml', 'w', encoding='utf-8') as config_file:
         config_file.write('allowed_resync_max_size:\n')
-        config_file.write(f'  table_bytes: {table_byte}\n')
+        config_file.write(f'  table_mb: {table_mb}\n')
 
     tasks.run_command(f'pipelinewise import_config --dir {TEST_PROJECTS_DIR_PATH}')
 

diff --git a/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py
@@ -9,10 +9,10 @@
 TARGET_ID = 'snowflake'
 
 
-def _create_ppw_config_file(table_byte):
+def _create_ppw_config_file(table_mb):
     with open(f'{TEST_PROJECTS_DIR_PATH}/config.yml', 'w', encoding='utf-8') as config_file:
         config_file.write('allowed_resync_max_size:\n')
-        config_file.write(f'  table_bytes: {table_byte}\n')
+        config_file.write(f'  table_mb: {table_mb}\n')
 
     tasks.run_command(f'pipelinewise import_config --dir {TEST_PROJECTS_DIR_PATH}')