Merge pull request #22 from eth-cscs/RESTAPI-1441-fix-issue-download-s3

Fix upload/download: added safe for URLs in jinja templates
eth-cscs · Feb 28, 2025 · de9eb0a · de9eb0a
2 parents 9becb27 + 0fd9b7f
commit de9eb0a
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [2.2.1]
+
+### Added
+
+### Changed
+
+### Fixed
+
+- Templates for upload and download using `filesystems/transfer` endpoint.
+
 ## [2.2.0]
 
 ### Added

diff --git a/src/firecrest/filesystem/transfer/scripts/slurm_job_downloader.sh b/src/firecrest/filesystem/transfer/scripts/slurm_job_downloader.sh
@@ -11,7 +11,7 @@ echo $(date -u) "Ingress File Transfer Job (id:${SLURM_JOB_ID})"
 echo $(date -u) "Waiting till file to tranfer is available..."
 for i in `seq 1440` 
 do 
-    status=$(curl --silent --head -o /dev/null --silent -Iw '%{http_code}' "{{ download_head_url }}")
+    status=$(curl --silent --head -o /dev/null --silent -Iw '%{http_code}' "{{ download_head_url | safe }}")
     if [[ "$status" == '200' ]]
         then
             echo $(date -u) "File to transfer found in S3 bucket"
@@ -33,7 +33,7 @@ do
                 part_file="$target_file.$part_i"
                 range_to=$(( range_from + range_length - 1 ))
 
-                http_code=$(curl --compressed --silent -D "$headers_file" --output "$part_file" --range "$range_from-$range_to" -w "%{http_code}" "{{ download_url }}")
+                http_code=$(curl --compressed --silent -D "$headers_file" --output "$part_file" --range "$range_from-$range_to" -w "%{http_code}" "{{ download_url | safe }}")
                 content_range=$(grep -i "Content-Range" "$headers_file")
                 file_length=$(echo ${content_range##*/} | tr -cd '[:digit:]')
                 content_length=$(grep -i "Content-Length" "$headers_file")

diff --git a/src/firecrest/filesystem/transfer/scripts/slurm_job_uploader_multipart.sh b/src/firecrest/filesystem/transfer/scripts/slurm_job_uploader_multipart.sh
@@ -14,7 +14,7 @@ BLOCK_SIZE=1048576
 MAX_PART_SIZE={{ F7T_MAX_PART_SIZE }}
 
 # Global parts array
-parts_url=({{ F7T_MP_PARTS_URL }})
+parts_url=({{ F7T_MP_PARTS_URL | safe }})
 
 # ----------------------------------------------------------------------------
 # UTILITY FUNCTIONS
@@ -174,7 +174,7 @@ parallel_run={{ F7T_MP_PARALLEL_RUN }}
 use_split={{ F7T_MP_USE_SPLIT }}
 num_parts={{ F7T_MP_NUM_PARTS }}
 input_file={{ F7T_MP_INPUT_FILE }}
-complete_multipart_url='{{ F7T_MP_COMPLETE_URL }}'
+complete_multipart_url='{{ F7T_MP_COMPLETE_URL | safe }}'
 
 echo "[INFO] Uploading file:$input_file into $num_parts chunks"