Merge pull request #2 from transferwise/AP-221

Remove undesired _SDC columns
transferwise · Jun 7, 2019 · b06579e · b06579e
2 parents ec07ac6 + 8f79a60
commit b06579e
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 18 deletions.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name="pipelinewise-target-snowflake",
-      version="1.0.0",
+      version="1.0.1",
       description="Singer.io target for loading data to Snowflake - PipelineWise compatible",
       author="TransferWise",
       url='https://github.com/transferwise/pipelinewise-target-snowflake',

diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py
@@ -41,13 +41,9 @@ def add_metadata_columns_to_schema(schema_message):
     Metadata columns gives information about data injections
     """
     extended_schema_message = schema_message
-    extended_schema_message['schema']['properties']['_sdc_batched_at'] = { 'type': ['null', 'string'], 'format': 'date-time' }
-    extended_schema_message['schema']['properties']['_sdc_deleted_at'] = { 'type': ['null', 'string'] }
     extended_schema_message['schema']['properties']['_sdc_extracted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' }
-    extended_schema_message['schema']['properties']['_sdc_primary_key'] = {'type': ['null', 'string'] }
-    extended_schema_message['schema']['properties']['_sdc_received_at'] = { 'type': ['null', 'string'], 'format': 'date-time' }
-    extended_schema_message['schema']['properties']['_sdc_sequence'] = {'type': ['integer'] }
-    extended_schema_message['schema']['properties']['_sdc_table_version'] = {'type': ['null', 'string'] }
+    extended_schema_message['schema']['properties']['_sdc_batched_at'] = { 'type': ['null', 'string'], 'format': 'date-time' }
+    extended_schema_message['schema']['properties']['_sdc_deleted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' }
 
     return extended_schema_message
 
@@ -56,13 +52,9 @@ def add_metadata_values_to_record(record_message, stream_to_sync):
     The location of the required attributes are fixed in the stream
     """
     extended_record = record_message['record']
+    extended_record['_sdc_extracted_at'] = record_message.get('time_extracted')
     extended_record['_sdc_batched_at'] = datetime.now().isoformat()
     extended_record['_sdc_deleted_at'] = record_message.get('record', {}).get('_sdc_deleted_at')
-    extended_record['_sdc_extracted_at'] = record_message.get('time_extracted')
-    extended_record['_sdc_primary_key'] = stream_to_sync.stream_schema_message['key_properties']
-    extended_record['_sdc_received_at'] = datetime.now().isoformat()
-    extended_record['_sdc_sequence'] = int(round(time.time() * 1000))
-    extended_record['_sdc_table_version'] = record_message.get('version')
 
     return extended_record
 

diff --git a/tests/integration/test_target_snowflake.py b/tests/integration/test_target_snowflake.py
@@ -17,13 +17,9 @@
 
 
 METADATA_COLUMNS = [
-    '_SDC_BATCHED_AT',
-    '_SDC_DELETED_AT',
     '_SDC_EXTRACTED_AT',
-    '_SDC_PRIMARY_KEY',
-    '_SDC_RECEIVED_AT',
-    '_SDC_SEQUENCE',
-    '_SDC_TABLE_VERSION'
+    '_SDC_BATCHED_AT',
+    '_SDC_DELETED_AT'
 ]