Skip to content
This repository has been archived by the owner on Sep 23, 2024. It is now read-only.

tests and proposed fix for tab char data corruption #422

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

tests and proposed fix for tab char data corruption #422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6db9761
tests and proposed fix for tab char data corruption
Feb 11, 2024
d057757
explicit str cast for bool etc
zugnush Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion target_snowflake/file_formats/csv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def record_to_csv_line(record: dict,

return ','.join(
[
json.dumps(flatten_record[column], ensure_ascii=False) if column in flatten_record and (
'"{}"'.format(str(flatten_record[column]).replace('"','""')) if column in flatten_record and (
flatten_record[column] == 0 or flatten_record[column]) else ''
for column in schema
]
Expand Down
6 changes: 5 additions & 1 deletion tests/unit/file_formats/test_csv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ def test_record_to_csv_line(self):
'key4': '25:01:01',
'key5': 'I\'m good',
'key6': None,
'key7': 'tab\tin\tvalue\n\r\0"ὠ',
}

schema = {
Expand Down Expand Up @@ -89,10 +90,13 @@ def test_record_to_csv_line(self):
'key6': {
'type': ['null', 'string'], 'format': 'time',
},
'key7':{
'type':['null', 'string'],
},
}

self.assertEqual(csv.record_to_csv_line(record, schema),
'"1","2030-01-22","10000-01-22 12:04:22","25:01:01","I\'m good",')
'"1","2030-01-22","10000-01-22 12:04:22","25:01:01","I\'m good",,"tab\tin\tvalue\n\r\0""ὠ"')

def test_create_copy_sql(self):
self.assertEqual(csv.create_copy_sql(table_name='foo_table',
Expand Down
6 changes: 5 additions & 1 deletion tests/unit/file_formats/test_parquet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ def test_records_to_dataframe(self):
'key4': '12:01:01',
'key5': 'I\'m good',
'key6': None,
'key7': 'A tab is a char too',
},
'2': {
'key1': 2,
Expand All @@ -29,6 +30,7 @@ def test_records_to_dataframe(self):
'key4': '13:01:01',
'key5': 'I\'m good too',
'key6': None,
'key7': 'A\tis a char too',
},
'3': {
'key1': 3,
Expand All @@ -37,6 +39,7 @@ def test_records_to_dataframe(self):
'key4': '14:01:01',
'key5': 'I want to be good',
'key6': None,
'key7': 'A\t\tis a char too',
}
}

Expand All @@ -48,7 +51,8 @@ def test_records_to_dataframe(self):
'key3': ['10000-01-22 12:04:22', '10000-01-22 12:04:22', '10000-01-22 12:04:22'],
'key4': ['12:01:01', '13:01:01', '14:01:01'],
'key5': ['I\'m good', 'I\'m good too', 'I want to be good'],
'key6': [None, None, None]}))
'key6': [None, None, None],
'key7': ['A tab is a char too', 'A\tis a char too', 'A\t\tis a char too'] }))

def test_create_copy_sql(self):
self.assertEqual(parquet.create_copy_sql(table_name='foo_table',
Expand Down