Copy existing metadata when archiving load files (#189)

transferwise · Jul 15, 2021 · 3c559dd · 3c559dd
1 parent 6b0b424
commit 3c559dd
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py
@@ -414,8 +414,7 @@ def copy_to_archive(self, s3_source_key, s3_archive_key, s3_archive_metadata):
                         As destination bucket, the config value 'archive_load_files_s3_bucket' will be used. If none is
                         specified, the bucket configured as 's3_bucket' will be used.
 
-        s3_archive_metadata: This dict will be used as the S3 metadata in the file in archive destination. Metadata in
-                             the source file will be replaced.
+        s3_archive_metadata: This dict will be merged with any metadata in the source file.
 
         """
         source_bucket = self.connection_config.get('s3_bucket')

diff --git a/target_snowflake/upload_clients/s3_upload_client.py b/target_snowflake/upload_clients/s3_upload_client.py
@@ -101,6 +101,9 @@ def delete_object(self, stream: str, key: str) -> None:
     def copy_object(self, copy_source: str, target_bucket: str, target_key: str, target_metadata: dict) -> None:
         """Copy object to another location on S3"""
         self.logger.info('Copying %s to %s/%s', copy_source, target_bucket, target_key)
+        source_bucket, source_key = copy_source.split("/", 1)
+        metadata = self.s3_client.head_object(Bucket=source_bucket, Key=source_key).get('Metadata', {})
+        metadata.update(target_metadata)
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.copy_object
         self.s3_client.copy_object(CopySource=copy_source, Bucket=target_bucket, Key=target_key,
-                                   Metadata=target_metadata, MetadataDirective="REPLACE")
+                                   Metadata=metadata, MetadataDirective="REPLACE")