From 3c559ddb397e139dc544fc91ad530bb1cac6c9e4 Mon Sep 17 00:00:00 2001 From: Lauri Lehtinen Date: Thu, 15 Jul 2021 17:01:22 +0300 Subject: [PATCH] Copy existing metadata when archiving load files (#189) --- target_snowflake/db_sync.py | 3 +-- target_snowflake/upload_clients/s3_upload_client.py | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py index 7f116b8a..32c6fc69 100644 --- a/target_snowflake/db_sync.py +++ b/target_snowflake/db_sync.py @@ -414,8 +414,7 @@ def copy_to_archive(self, s3_source_key, s3_archive_key, s3_archive_metadata): As destination bucket, the config value 'archive_load_files_s3_bucket' will be used. If none is specified, the bucket configured as 's3_bucket' will be used. - s3_archive_metadata: This dict will be used as the S3 metadata in the file in archive destination. Metadata in - the source file will be replaced. + s3_archive_metadata: This dict will be merged with any metadata in the source file. """ source_bucket = self.connection_config.get('s3_bucket') diff --git a/target_snowflake/upload_clients/s3_upload_client.py b/target_snowflake/upload_clients/s3_upload_client.py index 44fbfb5a..10801e59 100644 --- a/target_snowflake/upload_clients/s3_upload_client.py +++ b/target_snowflake/upload_clients/s3_upload_client.py @@ -101,6 +101,9 @@ def delete_object(self, stream: str, key: str) -> None: def copy_object(self, copy_source: str, target_bucket: str, target_key: str, target_metadata: dict) -> None: """Copy object to another location on S3""" self.logger.info('Copying %s to %s/%s', copy_source, target_bucket, target_key) + source_bucket, source_key = copy_source.split("/", 1) + metadata = self.s3_client.head_object(Bucket=source_bucket, Key=source_key).get('Metadata', {}) + metadata.update(target_metadata) # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.copy_object self.s3_client.copy_object(CopySource=copy_source, Bucket=target_bucket, Key=target_key, - Metadata=target_metadata, MetadataDirective="REPLACE") + Metadata=metadata, MetadataDirective="REPLACE")