improve upload to cos operator

component-library/filter/filter.cwl

@@ -6,7 +6,7 @@ baseCommand: "claimed"
 inputs:
   component:
     type: string
-    default: romeokienzler/claimed-filter:0.5
+    default: romeokienzler/claimed-filter:0.6
     inputBinding:
       position: 1
       prefix: --component
@@ -35,8 +35,5 @@ inputs:
       position: 5
       prefix: --output_file_name
 
-outputs:
-  dummy_out:
-    type: File
-    outputBinding:
-      glob: query_result.csv
+
+outputs: []

component-library/filter/filter.job.yaml

@@ -7,9 +7,9 @@ spec:
     spec:
       containers:
       - name: filter
-        image: romeokienzler/claimed-filter:0.5
+        image: romeokienzler/claimed-filter:0.6
         workingDir: /opt/app-root/src/
-        command: ["/opt/app-root/bin/ipython","filter.py"]
+        command: ["/opt/app-root/bin/ipython","claimed_filter.ipynb"]
         env:
         - name: log_level
           value: value_of_log_level

component-library/filter/filter.yaml

@@ -1,5 +1,5 @@
 name: output_file_name
-description: "# filter Filters rows based on predicate on pandas data frame Example 'predicate=~metadata.filename.str.contains('.gz') ' => filters all rows where column 'filename' contains '.gz' – CLAIMED V0.1"
+description: "# filter Filters rows based on predicate on pandas data frame  – CLAIMED V0.1"
 
 inputs:
 - {name: log_level, type: String, description: "update log level", default: "INFO"}
@@ -13,12 +13,12 @@ outputs:
 
 implementation:
     container:
-        image: romeokienzler/claimed-output_file_name:0.5
+        image: romeokienzler/claimed-output_file_name:0.6
         command:
         - sh
         - -ec
         - |
-          ipython ./filter.py log_level="${0}" predicate="${1}" file_name="${2}" output_file_name="${3}" 
+          ipython ./claimed_filter.ipynb log_level="${0}" predicate="${1}" file_name="${2}" output_file_name="${3}" 
         - {inputValue: log_level}
         - {inputValue: predicate}
         - {inputValue: file_name}

component-library/output/upload-to-cos.cwl

@@ -0,0 +1,45 @@
+cwlVersion: v1.2
+class: CommandLineTool
+
+baseCommand: "claimed"
+
+inputs:
+  component:
+    type: string
+    default: romeokienzler/claimed-upload-to-cos:0.8
+    inputBinding:
+      position: 1
+      prefix: --component
+  log_level:
+    type: string
+    default: "INFO"
+    inputBinding:
+      position: 2
+      prefix: --log_level
+  target:
+    type: string
+    default: None
+    inputBinding:
+      position: 3
+      prefix: --target
+  source_file_pattern:
+    type: string
+    default: None
+    inputBinding:
+      position: 4
+      prefix: --source_file_pattern
+  find_recursive:
+    type: bool
+    default: True
+    inputBinding:
+      position: 5
+      prefix: --find_recursive
+  process_target_file_pattern:
+    type: string
+    default: None
+    inputBinding:
+      position: 6
+      prefix: --process_target_file_pattern
+
+
+outputs: []

component-library/output/upload-to-cos.ipynb

@@ -41,7 +41,7 @@
    },
    "outputs": [],
    "source": [
-    "#!pip install s3fs"
+    "#!pip install aiobotocore botocore s3fs"
    ]
   },
   {
@@ -186,7 +186,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.11.7"
   },
   "papermill": {
    "default_parameters": {},

component-library/output/upload-to-cos.job.yaml

@@ -0,0 +1,26 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: upload-to-cos
+spec:
+  template:
+    spec:
+      containers:
+      - name: upload-to-cos
+        image: romeokienzler/claimed-upload-to-cos:0.8
+        workingDir: /opt/app-root/src/
+        command: ["/opt/app-root/bin/ipython","claimed_upload-to-cos.ipynb"]
+        env:
+        - name: log_level
+          value: value_of_log_level
+        - name: target
+          value: value_of_target
+        - name: source_file_pattern
+          value: value_of_source_file_pattern
+        - name: find_recursive
+          value: value_of_find_recursive
+        - name: process_target_file_pattern
+          value: value_of_process_target_file_pattern
+      restartPolicy: OnFailure
+      imagePullSecrets:
+        - name: image_pull_secret

component-library/output/upload-to-cos.yaml

@@ -1,33 +1,27 @@
-name: output-upload-to-cos
-description: Uploads a file to any S3 compliant Cloud Object Storage CLAIMED v0.2n
+name: process_target_file_pattern
+description: "# output-upload-to-cos Uploads a file to any S3 compliant Cloud Object Storage – CLAIMED V0.1"
 
 inputs:
-- {name: access_key_id, type: String, description: 'access key id'}
-- {name: secret_access_key, type: String, description: 'secret access key'}
-- {name: endpoint, type: String, description: 'cos/s3 endpoint'}
-- {name: bucket_name, type: String, description: 'cos bucket name'}
-- {name: source_file, type: OutputPath, description: 'source file to be uploaded'}
-- {name: destination_file, type: String, description: 'destination file name'}
-- {name: data_dir, type: String, description: 'temporary data folder'}
+- {name: log_level, type: String, description: "update log level", default: "INFO"}
+- {name: target, type: String, description: "target in format: cos://access_key_id:secret_access_key@endpoint/bucket/path"}
+- {name: source_file_pattern, type: String, description: "source folder and file pattern (glob)"}
+- {name: find_recursive, type: Boolean, description: "find_recursive, if True, will search for files in subfolders specified in source_file_pattern. Default is True", default: "True'"}
+- {name: process_target_file_pattern, type: String, description: "process source file path on target using regex. Default is None", default: "None"}
 
 
 outputs:
-- {name: output_dummy, type: String, description: 'dummy_output (to be fixed once C3 supports < 1 outputs)'}
 
 
 implementation:
     container:
-        image: romeokienzler/claimed-output-upload-to-cos:0.2n
+        image: romeokienzler/claimed-process_target_file_pattern:0.8
         command:
         - sh
         - -ec
         - |
-          ipython ./upload-to-cos.ipynb output_dummy="$0" access_key_id="$1" secret_access_key="$2" endpoint="$3" bucket_name="$4" source_file="$5" destination_file="$6" data_dir="$7" 
-        - {outputPath: output_dummy}
-        - {inputValue: access_key_id}
-        - {inputValue: secret_access_key}
-        - {inputValue: endpoint}
-        - {inputValue: bucket_name}
-        - {inputValue: source_file}
-        - {inputValue: destination_file}
-        - {inputValue: data_dir}
+          ipython ./claimed_upload-to-cos.ipynb log_level="${0}" target="${1}" source_file_pattern="${2}" find_recursive="${3}" process_target_file_pattern="${4}" 
+        - {inputValue: log_level}
+        - {inputValue: target}
+        - {inputValue: source_file_pattern}
+        - {inputValue: find_recursive}
+        - {inputValue: process_target_file_pattern}