From ab2981270c14f3ec7fcd230b99fdb1af5848bbed Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Fri, 7 Jun 2019 14:55:22 +0100 Subject: [PATCH] Remove undesired _SDC columns --- target_snowflake/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py index 5e358537..e2a39beb 100644 --- a/target_snowflake/__init__.py +++ b/target_snowflake/__init__.py @@ -41,9 +41,9 @@ def add_metadata_columns_to_schema(schema_message): Metadata columns gives information about data injections """ extended_schema_message = schema_message + extended_schema_message['schema']['properties']['_sdc_extracted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } extended_schema_message['schema']['properties']['_sdc_batched_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } extended_schema_message['schema']['properties']['_sdc_deleted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } - extended_schema_message['schema']['properties']['_sdc_extracted_at'] = { 'type': ['null', 'string'], 'format': 'date-time' } return extended_schema_message @@ -52,9 +52,9 @@ def add_metadata_values_to_record(record_message, stream_to_sync): The location of the required attributes are fixed in the stream """ extended_record = record_message['record'] + extended_record['_sdc_extracted_at'] = record_message.get('time_extracted') extended_record['_sdc_batched_at'] = datetime.now().isoformat() extended_record['_sdc_deleted_at'] = record_message.get('record', {}).get('_sdc_deleted_at') - extended_record['_sdc_extracted_at'] = record_message.get('time_extracted') return extended_record