From af5a3260169464a8323a9d35e0426c20347d1dff Mon Sep 17 00:00:00 2001 From: Peter Kosztolanyi Date: Mon, 20 Apr 2020 11:32:53 +0100 Subject: [PATCH] [AP-659] Fix loading tables with space in the name (#73) --- target_snowflake/db_sync.py | 4 ++-- .../messages-with-space-in-table-name.json | 16 ++++++++++++++++ tests/integration/test_target_snowflake.py | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/integration/resources/messages-with-space-in-table-name.json diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py index ba22212a..d06051be 100644 --- a/target_snowflake/db_sync.py +++ b/target_snowflake/db_sync.py @@ -438,7 +438,7 @@ def load_csv(self, s3_key, count, size_bytes): merge_sql = """MERGE INTO {} t USING ( SELECT {} - FROM @{}/{} + FROM '@{}/{}' (FILE_FORMAT => '{}')) s ON {} WHEN MATCHED THEN @@ -468,7 +468,7 @@ def load_csv(self, s3_key, count, size_bytes): # Insert only with COPY command if no primary key else: - copy_sql = """COPY INTO {} ({}) FROM @{}/{} + copy_sql = """COPY INTO {} ({}) FROM '@{}/{}' FILE_FORMAT = (format_name='{}') """.format( self.table_name(stream, False), diff --git a/tests/integration/resources/messages-with-space-in-table-name.json b/tests/integration/resources/messages-with-space-in-table-name.json new file mode 100644 index 00000000..b10f4400 --- /dev/null +++ b/tests/integration/resources/messages-with-space-in-table-name.json @@ -0,0 +1,16 @@ +{"type": "STATE", "value": {"currently_syncing": "my_db-table with space and UPPERCase"}} +{"type": "SCHEMA", "stream": "my_db-table with space and UPPERCase", "schema": {"properties": {"data": {"inclusion": "available", "format": "binary", "type": ["null", "string"]}, "id": {"inclusion": "automatic", "format": "binary", "type": ["null", "string"]}, "created_at": {"inclusion": "available", "format": "date-time", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["id"]} +{"type": "ACTIVATE_VERSION", "stream": "my_db-table with space and UPPERCase", "version": 1576670613163} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"data": "6461746132", "id": "706b32", "created_at": "2019-12-17T16:02:55+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T12:03:33.174343Z"} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"data": "64617461313030", "id": "706b33", "created_at": "2019-12-18T11:46:38+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T12:03:33.174343Z"} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"data": "6461746134", "id": "706b34", "created_at": "2019-12-17T16:32:22+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T12:03:33.174343Z"} +{"type": "STATE", "value": {"currently_syncing": "my_db-table with space and UPPERCase", "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163}}}} +{"type": "ACTIVATE_VERSION", "stream": "my_db-table with space and UPPERCase", "version": 1576670613163} +{"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163, "log_file": "mysql-bin.000004", "log_pos": 945}}}} +{"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163, "log_file": "mysql-bin.000004", "log_pos": 945}}}} +{"type": "SCHEMA", "stream": "my_db-table with space and UPPERCase", "schema": {"properties": {"data": {"inclusion": "available", "format": "binary", "type": ["null", "string"]}, "created_at": {"inclusion": "available", "format": "date-time", "type": ["null", "string"]}, "id": {"inclusion": "automatic", "format": "binary", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["id"]} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b35", "data": "6461746135", "created_at": "2019-12-18T13:19:20+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b35", "data": "64617461313030", "created_at": "2019-12-18T13:19:35+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b33", "data": "64617461313030", "created_at": "2019-12-18T11:46:38+00:00", "_sdc_deleted_at": "2019-12-18T13:19:44+00:00+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"} +{"type": "RECORD", "stream": "my_db-table with space and UPPERCase", "record": {"id": "706b35", "data": "64617461313030", "created_at": "2019-12-18T13:19:35+00:00", "_sdc_deleted_at": "2019-12-18T13:19:44+00:00+00:00"}, "version": 1576670613163, "time_extracted": "2019-12-18T13:24:31.441849Z"} +{"type": "STATE", "value": {"currently_syncing": null, "bookmarks": {"my_db-table with space and UPPERCase": {"version": 1576670613163, "log_file": "mysql-bin.000004", "log_pos": 1867}}}} diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py index 8f9a0d11..b3ee49fc 100644 --- a/tests/integration/test_target_snowflake.py +++ b/tests/integration/test_target_snowflake.py @@ -383,6 +383,20 @@ def test_loading_table_with_reserved_word_as_name_and_hard_delete(self): should_metadata_columns_exist=True ) + def test_loading_table_with_space(self): + """Loading a table where the name has space""" + tap_lines = test_utils.get_test_tap_lines('messages-with-space-in-table-name.json') + + # Turning on hard delete mode + self.config['hard_delete'] = True + self.persist_lines_with_cache(tap_lines) + + # Check if data loaded correctly and metadata columns exist + self.assert_binary_data_are_in_snowflake( + table_name='"TABLE WITH SPACE AND UPPERCASE"', + should_metadata_columns_exist=True + ) + def test_loading_unicode_characters(self): """Loading unicode encoded characters""" tap_lines = test_utils.get_test_tap_lines('messages-with-unicode-characters.json')