diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py index 38c0b638..b5aff846 100644 --- a/target_snowflake/db_sync.py +++ b/target_snowflake/db_sync.py @@ -82,6 +82,8 @@ def column_type(schema_property): # Every date-time JSON value is currently mapped to TIMESTAMP_NTZ elif property_format == 'date-time': col_type = 'timestamp_ntz' + elif property_format == 'date': + col_type = 'date' elif property_format == 'time': col_type = 'time' elif property_format == 'binary': diff --git a/tests/integration/resources/messages-with-multi-schemas.json b/tests/integration/resources/messages-with-multi-schemas.json index 993e2587..9f3cc6fd 100644 --- a/tests/integration/resources/messages-with-multi-schemas.json +++ b/tests/integration/resources/messages-with-multi-schemas.json @@ -6,10 +6,10 @@ {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}}}} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_two": {"initial_full_table_complete": true}}}} -{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_two", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_date": {"format": "date-time", "inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} +{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_two", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_date": {"format": "date-time", "inclusion": "available", "type": ["null", "string"]}, "c_iso_date": {"format": "date", "inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_two", "version": 3} -{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1, "c_date": "2019-02-01 15:12:45", "_sdc_deleted_at": "2019-02-12T01:10:10.000000Z"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} -{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} +{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1, "c_date": "2019-02-01 15:12:45", "c_iso_date": "2019-02-01", "_sdc_deleted_at": "2019-02-12T01:10:10.000000Z"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} +{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00", "c_iso_date": "2019-02-10"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_wo": {"initial_full_table_complete": true}}}} {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_three", "version": 3} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}, "tap_mysql_test-test_table_three": {"initial_full_table_complete": true}}}} diff --git a/tests/integration/resources/messages-with-three-streams-modified-column.json b/tests/integration/resources/messages-with-three-streams-modified-column.json index f0acee87..4867b631 100644 --- a/tests/integration/resources/messages-with-three-streams-modified-column.json +++ b/tests/integration/resources/messages-with-three-streams-modified-column.json @@ -6,10 +6,10 @@ {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}}}} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_two": {"initial_full_table_complete": true}}}} -{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_two", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_date": {"inclusion": "available", "type": ["null", "string"]}, "c_new_column": {"inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} +{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_two", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_date": {"inclusion": "available", "type": ["null", "string"]}, "c_iso_date": {"format": "date", "inclusion": "available", "type": ["null", "string"]}, "c_new_column": {"inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_two", "version": 3} -{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-12 02:00:00", "c_new_column": "data 1"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} -{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 3, "c_varchar": "2", "c_int": 3, "c_date": "2019-02-15 02:00:00", "c_new_column": "data 2"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} +{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-12 02:00:00", "c_iso_date": "2019-02-10", "c_new_column": "data 1"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} +{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 3, "c_varchar": "2", "c_int": 3, "c_date": "2019-02-15 02:00:00", "c_iso_date": "2019-02-15", "c_new_column": "data 2"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_wo": {"initial_full_table_complete": true}}}} {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_three", "version": 3} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}, "tap_mysql_test-test_table_three": {"initial_full_table_complete": true}}}} diff --git a/tests/integration/resources/messages-with-three-streams.json b/tests/integration/resources/messages-with-three-streams.json index 975cc0f3..75b49eba 100644 --- a/tests/integration/resources/messages-with-three-streams.json +++ b/tests/integration/resources/messages-with-three-streams.json @@ -6,10 +6,10 @@ {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_one", "version": 1} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_one", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}}}} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_two": {"initial_full_table_complete": true}}}} -{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_two", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_date": {"format": "date-time", "inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} +{"type": "SCHEMA", "stream": "tap_mysql_test-test_table_two", "schema": {"properties": {"c_pk": {"inclusion": "automatic", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_varchar": {"inclusion": "available", "maxLength": 16, "type": ["null", "string"]}, "c_int": {"inclusion": "available", "minimum": -2147483648, "maximum": 2147483647, "type": ["null", "integer"]}, "c_date": {"format": "date-time", "inclusion": "available", "type": ["null", "string"]}, "c_iso_date": {"format": "date", "inclusion": "available", "type": ["null", "string"]}}, "type": "object"}, "key_properties": ["c_pk"]} {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_two", "version": 3} -{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1, "c_date": "2019-02-01 15:12:45", "_sdc_deleted_at": "2019-02-12T01:10:10.000000Z"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} -{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} +{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 1, "c_varchar": "1", "c_int": 1, "c_date": "2019-02-01 15:12:45", "c_iso_date": "2019-02-01", "_sdc_deleted_at": "2019-02-12T01:10:10.000000Z"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} +{"type": "RECORD", "stream": "tap_mysql_test-test_table_two", "record": {"c_pk": 2, "c_varchar": "2", "c_int": 2, "c_date": "2019-02-10 02:00:00", "c_iso_date": "2019-02-10"}, "version": 3, "time_extracted": "2019-01-31T15:51:48.861962Z"} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_wo": {"initial_full_table_complete": true}}}} {"type": "ACTIVATE_VERSION", "stream": "tap_mysql_test-test_table_three", "version": 3} {"type": "STATE", "value": {"currently_syncing": "tap_mysql_test-test_table_two", "bookmarks": {"tap_mysql_test-test_table_one": {"initial_full_table_complete": true}, "tap_mysql_test-test_table_three": {"initial_full_table_complete": true}}}} diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py index fda8508c..4a293e24 100644 --- a/tests/integration/test_target_snowflake.py +++ b/tests/integration/test_target_snowflake.py @@ -138,17 +138,17 @@ def assert_three_streams_are_into_snowflake(self, should_metadata_columns_exist= self.remove_metadata_columns_from_rows(table_one), expected_table_one) # ---------------------------------------------------------------------- - # Check rows in table_tow + # Check rows in table_two # ---------------------------------------------------------------------- expected_table_two = [] if not should_hard_deleted_rows: expected_table_two = [ - {'C_INT': 1, 'C_PK': 1, 'C_VARCHAR': '1', 'C_DATE': datetime.datetime(2019, 2, 1, 15, 12, 45)}, - {'C_INT': 2, 'C_PK': 2, 'C_VARCHAR': '2', 'C_DATE': datetime.datetime(2019, 2, 10, 2, 0, 0)} + {'C_INT': 1, 'C_PK': 1, 'C_VARCHAR': '1', 'C_DATE': datetime.datetime(2019, 2, 1, 15, 12, 45), 'C_ISO_DATE':datetime.date(2019, 2, 1)}, + {'C_INT': 2, 'C_PK': 2, 'C_VARCHAR': '2', 'C_DATE': datetime.datetime(2019, 2, 10, 2, 0, 0), 'C_ISO_DATE':datetime.date(2019, 2, 10)} ] else: expected_table_two = [ - {'C_INT': 2, 'C_PK': 2, 'C_VARCHAR': '2', 'C_DATE': datetime.datetime(2019, 2, 10, 2, 0, 0)} + {'C_INT': 2, 'C_PK': 2, 'C_VARCHAR': '2', 'C_DATE': datetime.datetime(2019, 2, 10, 2, 0, 0), 'C_ISO_DATE':datetime.date(2019, 2, 10)} ] self.assertEqual( @@ -205,7 +205,7 @@ def assert_logical_streams_are_in_snowflake(self, should_metadata_columns_exist= ] # ---------------------------------------------------------------------- - # Check rows in table_tow + # Check rows in table_two # ---------------------------------------------------------------------- expected_table_two = [ {'CID': 1, 'CVARCHAR': "updated row"}, @@ -600,11 +600,11 @@ def test_column_name_change(self): table_two, [ {previous_column_name: datetime.datetime(2019, 2, 1, 15, 12, 45), 'C_INT': 1, 'C_PK': 1, - 'C_VARCHAR': '1', 'C_DATE': None, 'C_NEW_COLUMN': None}, + 'C_VARCHAR': '1', 'C_DATE': None, 'C_ISO_DATE': datetime.date(2019, 2, 1), 'C_NEW_COLUMN': None}, {previous_column_name: datetime.datetime(2019, 2, 10, 2), 'C_INT': 2, 'C_PK': 2, 'C_VARCHAR': '2', - 'C_DATE': '2019-02-12 02:00:00', 'C_NEW_COLUMN': 'data 1'}, + 'C_DATE': '2019-02-12 02:00:00', 'C_ISO_DATE': datetime.date(2019, 2, 10), 'C_NEW_COLUMN': 'data 1'}, {previous_column_name: None, 'C_INT': 3, 'C_PK': 3, 'C_VARCHAR': '2', 'C_DATE': '2019-02-15 02:00:00', - 'C_NEW_COLUMN': 'data 2'} + 'C_ISO_DATE': datetime.date(2019, 2, 15), 'C_NEW_COLUMN': 'data 2'} ] ) @@ -661,12 +661,12 @@ def test_column_name_change_without_table_cache(self): table_two, [ {previous_column_name: datetime.datetime(2019, 2, 1, 15, 12, 45), 'C_INT': 1, 'C_PK': 1, - 'C_VARCHAR': '1', 'C_DATE': None, 'C_NEW_COLUMN': None}, + 'C_VARCHAR': '1', 'C_DATE': None, 'C_ISO_DATE': datetime.date(2019, 2, 1), 'C_NEW_COLUMN': None}, {previous_column_name: datetime.datetime(2019, 2, 10, 2), 'C_INT': 2, 'C_PK': 2, 'C_VARCHAR': '2', - 'C_DATE': '2019-02-12 02:00:00', 'C_NEW_COLUMN': 'data 1'}, + 'C_DATE': '2019-02-12 02:00:00', 'C_ISO_DATE': datetime.date(2019, 2, 10), 'C_NEW_COLUMN': 'data 1'}, {previous_column_name: None, 'C_INT': 3, 'C_PK': 3, 'C_VARCHAR': '2', 'C_DATE': '2019-02-15 02:00:00', - 'C_NEW_COLUMN': 'data 2'} - ] + 'C_ISO_DATE': datetime.date(2019, 2, 15), 'C_NEW_COLUMN': 'data 2'} + ] ) # Table three should have a renamed columns and a new column diff --git a/tests/unit/test_db_sync.py b/tests/unit/test_db_sync.py index ef2d1911..a22a2064 100644 --- a/tests/unit/test_db_sync.py +++ b/tests/unit/test_db_sync.py @@ -19,6 +19,8 @@ def setUp(self): 'str_or_null': {"type": ["string", "null"]}, 'dt': {"type": ["string"], "format": "date-time"}, 'dt_or_null': {"type": ["string", "null"], "format": "date-time"}, + 'd': {"type": ["string"], "format": "date"}, + 'd_or_null': {"type": ["string", "null"], "format": "date"}, 'time': {"type": ["string"], "format": "time"}, 'time_or_null': {"type": ["string", "null"], "format": "time"}, 'binary': {"type": ["string", "null"], "format": "binary"}, @@ -99,6 +101,8 @@ def test_column_type_mapping(self): 'str_or_null': 'text', 'dt': 'timestamp_ntz', 'dt_or_null': 'timestamp_ntz', + 'd': 'date', + 'd_or_null': 'date', 'time': 'time', 'time_or_null': 'time', 'binary': 'binary', @@ -124,6 +128,8 @@ def test_column_trans(self): 'str_or_null': '', 'dt': '', 'dt_or_null': '', + 'd': '', + 'd_or_null': '', 'time': '', 'time_or_null': '', 'binary': 'to_binary',