From 4c97e0c8d8be2a621bcb1c95ebe1606c0ab8d503 Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 7 Jun 2019 14:46:04 +0100 Subject: [PATCH 1/5] Remove undesired _SDC columns --- target_snowflake/__init__.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py index b08274e0..5e358537 100644 --- a/target_snowflake/__init__.py +++ b/target_snowflake/__init__.py @@ -42,12 +42,8 @@ def add_metadata_columns_to_schema(schema_message): """ extended_schema_message = schema_message extended_schema_message['schema']['properties']['_sdc_batched_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } - extended_schema_message['schema']['properties']['_sdc_deleted_at'] = { 'type': ['null', 'string'] } + extended_schema_message['schema']['properties']['_sdc_deleted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } extended_schema_message['schema']['properties']['_sdc_extracted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } - extended_schema_message['schema']['properties']['_sdc_primary_key'] = {'type': ['null', 'string'] } - extended_schema_message['schema']['properties']['_sdc_received_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } - extended_schema_message['schema']['properties']['_sdc_sequence'] = {'type': ['integer'] } - extended_schema_message['schema']['properties']['_sdc_table_version'] = {'type': ['null', 'string'] } return extended_schema_message @@ -59,10 +55,6 @@ def add_metadata_values_to_record(record_message, stream_to_sync): extended_record['_sdc_batched_at'] = datetime.now().isoformat() extended_record['_sdc_deleted_at'] = record_message.get('record', {}).get('_sdc_deleted_at') extended_record['_sdc_extracted_at'] = record_message.get('time_extracted') - extended_record['_sdc_primary_key'] = stream_to_sync.stream_schema_message['key_properties'] - extended_record['_sdc_received_at'] = datetime.now().isoformat() - extended_record['_sdc_sequence'] = int(round(time.time() * 1000)) - extended_record['_sdc_table_version'] = record_message.get('version') return extended_record From ab2981270c14f3ec7fcd230b99fdb1af5848bbed Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 7 Jun 2019 14:55:22 +0100 Subject: [PATCH 2/5] Remove undesired _SDC columns --- target_snowflake/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py index 5e358537..e2a39beb 100644 --- a/target_snowflake/__init__.py +++ b/target_snowflake/__init__.py @@ -41,9 +41,9 @@ def add_metadata_columns_to_schema(schema_message): Metadata columns gives information about data injections """ extended_schema_message = schema_message + extended_schema_message['schema']['properties']['_sdc_extracted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } extended_schema_message['schema']['properties']['_sdc_batched_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } extended_schema_message['schema']['properties']['_sdc_deleted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } - extended_schema_message['schema']['properties']['_sdc_extracted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } return extended_schema_message @@ -52,9 +52,9 @@ def add_metadata_values_to_record(record_message, stream_to_sync): The location of the required attributes are fixed in the stream """ extended_record = record_message['record'] + extended_record['_sdc_extracted_at'] = record_message.get('time_extracted') extended_record['_sdc_batched_at'] = datetime.now().isoformat() extended_record['_sdc_deleted_at'] = record_message.get('record', {}).get('_sdc_deleted_at') - extended_record['_sdc_extracted_at'] = record_message.get('time_extracted') return extended_record From c7271e34670fd67b838d899263d01f8cf740c8be Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 7 Jun 2019 15:18:14 +0100 Subject: [PATCH 3/5] Remove undesired _SDC columns --- tests/integration/test_target_snowflake.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py index 00d2cf9a..440de80c 100644 --- a/tests/integration/test_target_snowflake.py +++ b/tests/integration/test_target_snowflake.py @@ -17,13 +17,9 @@ METADATA_COLUMNS = [ + '_SDC_EXTRACTED_AT', '_SDC_BATCHED_AT', '_SDC_DELETED_AT', - '_SDC_EXTRACTED_AT', - '_SDC_PRIMARY_KEY', - '_SDC_RECEIVED_AT', - '_SDC_SEQUENCE', - '_SDC_TABLE_VERSION' ] From cfc59f62e6c22cca07783aa0a123ef8a7910e6ed Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 7 Jun 2019 15:19:01 +0100 Subject: [PATCH 4/5] Remove undesired _SDC columns --- tests/integration/test_target_snowflake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py index 440de80c..fb029aab 100644 --- a/tests/integration/test_target_snowflake.py +++ b/tests/integration/test_target_snowflake.py @@ -19,7 +19,7 @@ METADATA_COLUMNS = [ '_SDC_EXTRACTED_AT', '_SDC_BATCHED_AT', - '_SDC_DELETED_AT', + '_SDC_DELETED_AT' ] From 8f79a604b667160f4748a9abbcdac16ff66781b6 Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 7 Jun 2019 15:51:41 +0100 Subject: [PATCH 5/5] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ad0cdd34..28bc1915 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup(name="pipelinewise-target-snowflake", - version="1.0.0", + version="1.0.1", description="Singer.io target for loading data to Snowflake - PipelineWise compatible", author="TransferWise", url='https://github.com/transferwise/pipelinewise-target-snowflake',