[AP-498] Change default /tmp folder for encrypting files

transferwise · Jan 31, 2020 · 0e200f6 · 0e200f6
1 parent 9e1c55d
commit 0e200f6
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 6 deletions.
diff --git a/setup.py b/setup.py
@@ -6,7 +6,7 @@
       long_description = f.read()
 
 setup(name="pipelinewise-target-snowflake",
-      version="1.4.0",
+      version="1.4.1",
       description="Singer.io target for loading data to Snowflake - PipelineWise compatible",
       long_description=long_description,
       long_description_content_type='text/markdown',

diff --git a/target_snowflake/__init__.py b/target_snowflake/__init__.py
@@ -157,7 +157,6 @@ def persist_lines(config, lines, information_schema_cache=None) -> None:
     key_properties = {}
     validators = {}
     records_to_load = {}
-    csv_files_to_load = {}
     row_count = {}
     stream_to_sync = {}
     total_row_count = {}
@@ -292,7 +291,6 @@ def persist_lines(config, lines, information_schema_cache=None) -> None:
 
             row_count[stream] = 0
             total_row_count[stream] = 0
-            csv_files_to_load[stream] = NamedTemporaryFile(mode='w+b')
 
         elif t == 'ACTIVATE_VERSION':
             logger.debug('ACTIVATE_VERSION message')
@@ -416,7 +414,7 @@ def flush_records(stream, records_to_load, row_count, db_sync, temp_dir=None):
             csv_line = db_sync.record_to_csv_line(record)
             f.write(bytes(csv_line + '\n', 'UTF-8'))
 
-    s3_key = db_sync.put_to_stage(csv_file, stream, row_count)
+    s3_key = db_sync.put_to_stage(csv_file, stream, row_count, temp_dir=temp_dir)
     try:
         db_sync.load_csv(s3_key, row_count)
     except Exception as e:

diff --git a/target_snowflake/db_sync.py b/target_snowflake/db_sync.py
@@ -358,7 +358,7 @@ def record_to_csv_line(self, record):
             ]
         )
 
-    def put_to_stage(self, file, stream, count):
+    def put_to_stage(self, file, stream, count, temp_dir=None):
         logger.info("Uploading {} rows to external snowflake stage on S3".format(count))
 
         # Generating key in S3 bucket 
@@ -379,7 +379,8 @@ def put_to_stage(self, file, stream, count):
             )
             encryption_metadata, encrypted_file = SnowflakeEncryptionUtil.encrypt_file(
                 encryption_material,
-                file
+                file,
+                tmp_dir=temp_dir
             )
 
             # Upload to s3