[AP-659] Fix loading tables with space in the name (#73)

transferwise · Apr 20, 2020 · af5a326 · af5a326
1 parent 76cd31d
commit af5a326
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 2 deletions.
diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py
@@ -438,7 +438,7 @@ def load_csv(self, s3_key, count, size_bytes):
                     merge_sql = """MERGE INTO {} t
                         USING (
                             SELECT {}
-                              FROM @{}/{}
+                              FROM '@{}/{}'
                               (FILE_FORMAT => '{}')) s
                         ON {}
                         WHEN MATCHED THEN
@@ -468,7 +468,7 @@ def load_csv(self, s3_key, count, size_bytes):
 
                 # Insert only with COPY command if no primary key
                 else:
-                    copy_sql = """COPY INTO {} ({}) FROM @{}/{}
+                    copy_sql = """COPY INTO {} ({}) FROM '@{}/{}'
                         FILE_FORMAT = (format_name='{}')
                     """.format(
                         self.table_name(stream, False),

diff --git a/tests/integration/resources/messages-with-space-in-table-name.json b/tests/integration/resources/messages-with-space-in-table-name.json
@@ -0,0 +1,16 @@
+{"type": "STATE", "value": {"currently_syncing": "my_db-table with space and UPPERCase"}}
+{"type": "SCHEMA", "stream": "my_db-table with space and UPPERCase", "schema": {"properties": {"data": {"inclusion": "available", "format": "binary", "type": ["null", "string"]}, "id": {"inclusion": "automatic", "format": "binary", "type": ["null", "string"]}, "created_at": {"inclusion": "available", "format": "date-time", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["id"]}
+{"type": "ACTIVATE_VERSION", "stream": "my_db-table with space and UPPERCase", "version": 1576670613163}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"data": "6461746132", "id": "706b32", "created_at": "2019-12-17T16:02:55+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T12:03:33.174343Z"}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"data": "64617461313030", "id": "706b33", "created_at": "2019-12-18T11:46:38+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T12:03:33.174343Z"}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"data": "6461746134", "id": "706b34", "created_at": "2019-12-17T16:32:22+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T12:03:33.174343Z"}
+{"type": "STATE", "value": {"currently_syncing": "my_db-table with space and UPPERCase", "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163}}}}
+{"type": "ACTIVATE_VERSION", "stream": "my_db-table with space and UPPERCase", "version": 1576670613163}
+{"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163, "log_file": "mysql-bin.000004", "log_pos": 945}}}}
+{"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163, "log_file": "mysql-bin.000004", "log_pos": 945}}}}
+{"type": "SCHEMA", "stream": "my_db-table with space and UPPERCase", "schema": {"properties": {"data": {"inclusion": "available", "format": "binary", "type": ["null", "string"]}, "created_at": {"inclusion": "available", "format": "date-time", "type": ["null", "string"]}, "id": {"inclusion": "automatic", "format": "binary", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["id"]}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b35", "data": "6461746135", "created_at": "2019-12-18T13:19:20+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b35", "data": "64617461313030", "created_at": "2019-12-18T13:19:35+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b33", "data": "64617461313030", "created_at": "2019-12-18T11:46:38+00:00", "_sdc_deleted_at": "2019-12-18T13:19:44+00:00+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"}
+{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b35", "data": "64617461313030", "created_at": "2019-12-18T13:19:35+00:00", "_sdc_deleted_at": "2019-12-18T13:19:44+00:00+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"}
+{"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163, "log_file": "mysql-bin.000004", "log_pos": 1867}}}}
diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py
@@ -383,6 +383,20 @@ def test_loading_table_with_reserved_word_as_name_and_hard_delete(self):
             should_metadata_columns_exist=True
         )
 
+    def test_loading_table_with_space(self):
+        """Loading a table where the name has space"""
+        tap_lines = test_utils.get_test_tap_lines('messages-with-space-in-table-name.json')
+
+        # Turning on hard delete mode
+        self.config['hard_delete'] = True
+        self.persist_lines_with_cache(tap_lines)
+
+        # Check if data loaded correctly and metadata columns exist
+        self.assert_binary_data_are_in_snowflake(
+            table_name='"TABLE WITH SPACE AND UPPERCASE"',
+            should_metadata_columns_exist=True
+        )
+
     def test_loading_unicode_characters(self):
         """Loading unicode encoded characters"""
         tap_lines = test_utils.get_test_tap_lines('messages-with-unicode-characters.json')