changed setting to MB

transferwise · Jul 18, 2024 · bb74ac4 · bb74ac4
1 parent b4c9853
commit bb74ac4
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 13 deletions.
diff --git a/docs/user_guide/resync.rst b/docs/user_guide/resync.rst
@@ -53,7 +53,7 @@ add the ``--tables`` argument:
   .. code-block:: yaml
 
      allowed_resync_max_size:
-       table_bytes: <integer>
+       table_mb: <integer/float>
 
 
 

diff --git a/pipelinewise/fastsync/mysql_to_snowflake.py b/pipelinewise/fastsync/mysql_to_snowflake.py
@@ -215,7 +215,7 @@ def main_impl():
             all_tables_in_this_schema = get_tables_size(schema, tap_obj)
             only_selected_tables = filter_out_selected_tables(all_tables_in_this_schema, args.tables)
             table_with_maximum_size = get_maximum_value_from_list_of_dicts(only_selected_tables, 'table_size')
-            if table_with_maximum_size.get('table_size') > int(args.autoresync_size):
+            if table_with_maximum_size.get('table_size') > float(args.autoresync_size):
                 can_run_sync = False
                 table_sync_excs.append(
                     f're-sync can not be done because size of table '

diff --git a/pipelinewise/fastsync/postgres_to_snowflake.py b/pipelinewise/fastsync/postgres_to_snowflake.py
@@ -213,7 +213,7 @@ def main_impl():
             all_tables_in_this_schema = get_tables_size(schema, tap_obj)
             only_selected_tables = filter_out_selected_tables(all_tables_in_this_schema, args.tables)
             table_with_maximum_size = get_maximum_value_from_list_of_dicts(only_selected_tables, 'table_size')
-            if table_with_maximum_size.get('table_size') > int(args.autoresync_size):
+            if table_with_maximum_size.get('table_size') > float(args.autoresync_size):
                 can_run_sync = False
                 table_sync_excs.append(
                     f're-sync can not be done because size of table '

diff --git a/pipelinewise/utils.py b/pipelinewise/utils.py
@@ -31,7 +31,7 @@ def get_tables_size(schema: str, tap) -> dict:
         result_list = tap.query(
             'select TABLE_NAME as table_name,'
             ' TABLE_ROWS as table_rows,'
-            ' ROUND((DATA_LENGTH + INDEX_LENGTH)/ 1024 / 1024) as table_size'
+            ' (DATA_LENGTH + INDEX_LENGTH)/ 1024 / 1024 as table_size'
             f' from information_schema.TABLES where TABLE_SCHEMA = \'{schema}\';')
         tap.close_connections()
         for res in result_list:
@@ -48,7 +48,8 @@ def get_tables_size(schema: str, tap) -> dict:
             ' (xpath(\'/row/c/text()\','
             ' query_to_xml(format(\'select count(*) as c from %I.%I\', table_schema, TABLE_NAME), FALSE, TRUE, \'\'))'
             ')[1]::text::int AS table_rows,'
-            ' pg_total_relation_size(\'"\'||table_schema||\'"."\'||table_name||\'"\')/1024/1024 as table_size '
+            ' pg_total_relation_size('
+            '\'"\'||table_schema||\'"."\'||table_name||\'"\')::NUMERIC/1024::NUMERIC/1024 as table_size '
             'FROM (SELECT table_schema, TABLE_NAME FROM information_schema.tables '
             f'WHERE TABLE_NAME not like \'pg_%\' AND table_schema in (\'{schema}\')) as tb'
             )

diff --git a/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_mariadb/test_resync_mariadb_to_sf_table_size_check.py
@@ -32,7 +32,7 @@ def tearDown(self):
     def test_resync_mariadb_to_sf_if_table_size_greater_than_limit(self):  # pylint: disable = no-self-use
         """test resync mariadb to SF returns error 1 if table size is greater than the limit"""
 
-        a_small_number = 0.1   # Mb
+        a_small_number = 0.01   # Mb
         _create_ppw_config_file(table_mb=a_small_number)
 
         command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}'
@@ -43,7 +43,7 @@ def test_resync_mariadb_to_sf_if_table_size_greater_than_limit(self):  # pylint:
 
     def test_resync_mariadb_to_sf_if_table_size_less_than_limit(self):  # pylint: disable = no-self-use
         """test resync mariadb to SF returns error if table size is less than the limit"""
-        a_big_number = 100000000000
+        a_big_number = 10000 #Mb
         _create_ppw_config_file(table_mb=a_big_number)
 
         command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}'
@@ -53,7 +53,7 @@ def test_resync_mariadb_to_sf_if_table_size_less_than_limit(self):  # pylint: di
 
     def test_resync_mariadb_to_sf_if_table_size_greater_than_limit_and_force(self):  # pylint: disable = no-self-use
         """test resync mariadb to SF returns error if table size is greater than the limit and --force is used"""
-        a_small_number = 0.1  # Mb
+        a_small_number = 0.01  # Mb
         _create_ppw_config_file(table_mb=a_small_number)
 
         command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID} --force'
@@ -64,7 +64,7 @@ def test_resync_mariadb_to_sf_if_table_size_greater_than_limit_and_force(self):
 
     def test_run_tap_mariadb_to_sf_if_size_greater_than_limit(self):   # pylint: disable = no-self-use
         """test run_tap mariadb to sf if table size is greater than the limit"""
-        a_small_number = 0.1  # Mb
+        a_small_number = 0.01  # Mb
         _create_ppw_config_file(table_mb=a_small_number)
 
         command = f'pipelinewise run_tap --tap {TAP_ID} --target {TARGET_ID}'

diff --git a/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py b/tests/end_to_end/target_snowflake/tap_postgres/test_resync_pg_to_sf_table_size_check.py
@@ -32,7 +32,7 @@ def tearDown(self):
     def test_resync_pg_to_sf_if_table_size_greater_than_limit(self):   # pylint: disable = no-self-use
         """test resync pg to SF returns error 1 if table size is greater than the limit"""
 
-        a_small_number = 0.1   # Mb
+        a_small_number = 0.01   # Mb
         _create_ppw_config_file(table_mb=a_small_number)
 
         command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}'
@@ -43,7 +43,7 @@ def test_resync_pg_to_sf_if_table_size_greater_than_limit(self):   # pylint: dis
 
     def test_resync_pg_to_sf_if_table_size_less_than_limit(self):   # pylint: disable = no-self-use
         """test resync pg to SF returns error if table size is less than the limit"""
-        a_big_number = 100000000000
+        a_big_number = 1000 # Mb
         _create_ppw_config_file(table_mb=a_big_number)
 
         command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID}'
@@ -54,7 +54,7 @@ def test_resync_pg_to_sf_if_table_size_less_than_limit(self):   # pylint: disabl
     def test_resync_pg_to_sf_if_table_size_greater_than_limit_and_force(self):   # pylint: disable = no-self-use
 
         """test resync pg to SF returns error if table size is greater than the limit and --force is used"""
-        a_small_number = 0.1  # Mb
+        a_small_number = 0.01  # Mb
         _create_ppw_config_file(table_mb=a_small_number)
 
         command = f'pipelinewise sync_tables --tap {TAP_ID} --target {TARGET_ID} --force'
@@ -65,7 +65,7 @@ def test_resync_pg_to_sf_if_table_size_greater_than_limit_and_force(self):   # p
 
     def test_run_tap_pg_to_sf_if_size_greater_than_limit(self):   # pylint: disable = no-self-use
         """test run_tap postgres to sf if table size is greater than the limit"""
-        a_small_number = 0.1 # Mb
+        a_small_number = 0.01 # Mb
         _create_ppw_config_file(table_mb=a_small_number)
 
         command = f'pipelinewise run_tap --tap {TAP_ID} --target {TARGET_ID}'