Merge pull request #14 from shrutimantri/add-s3-compatible-examples-1

feat(docs): add examples for all minio tasks for s3-compatible storage
kestra-io · Aug 16, 2024 · 80336e6 · 80336e6
2 parents 8311ef9 + f3868e3
commit 80336e6
Show file tree

Hide file tree

Showing 8 changed files with 151 additions and 1 deletion.
diff --git a/src/main/java/io/kestra/plugin/minio/Copy.java b/src/main/java/io/kestra/plugin/minio/Copy.java
@@ -33,6 +33,26 @@
                 "  bucket: \"my-bucket2\"",
                 "  key: \"path/to/file2\"",
             }
+        ),
+        @Example(
+            title = "Copy file in an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_copy
+              namespace: company.team
+              tasks:
+                - id: "copy_file"
+                  type: "io.kestra.plugin.minio.Copy"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com
+                  from:
+                    bucket: "my-bucket"
+                    key: "path/to/file"
+                  to:
+                    bucket: "my-bucket2"
+                    key: "path/to/file2"
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/Delete.java b/src/main/java/io/kestra/plugin/minio/Delete.java
@@ -26,6 +26,22 @@
                 "bucket: \"my-bucket\"",
                 "key: \"path/to/file\""
             }
+        ),
+        @Example(
+            title = "Delete file from an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_delete
+              namespace: company.team
+              tasks:
+                - id: "delete"
+                  type: "io.kestra.plugin.minio.Downloads"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com
+                  bucket: "kestra-test-bucket"
+                  key: "path/to/file"
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/DeleteList.java b/src/main/java/io/kestra/plugin/minio/DeleteList.java
@@ -39,6 +39,22 @@
                 "bucket: \"my-bucket\"",
                 "prefix: \"sub-dir\""
             }
+        ),
+        @Example(
+            title = "Delete files from an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_delete_objects
+              namespace: company.team
+              tasks:
+                - id: "delete_objects"
+                  type: "io.kestra.plugin.minio.DeleteList"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com
+                  bucket: "kestra-test-bucket"
+                  prefix: "sub-dir"
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/Download.java b/src/main/java/io/kestra/plugin/minio/Download.java
@@ -36,6 +36,22 @@
                 "bucket: \"my-bucket\"",
                 "key: \"path/to/file\""
             }
+        ),
+        @Example(
+            title = "Download file from an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_download
+              namespace: company.team
+              tasks:
+                - id: "download_from_storage"
+                  type: "io.kestra.plugin.minio.Download"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com
+                  bucket: "kestra-test-bucket"
+                  key: "data/orders.csv"
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/Downloads.java b/src/main/java/io/kestra/plugin/minio/Downloads.java
@@ -34,8 +34,26 @@
                 "secretKeyId: \"<secret-key>\"",
                 "region: \"eu-central-1\"",
                 "bucket: \"my-bucket\"",
-                "prefix: \"sub-dir\""
+                "prefix: \"sub-dir\"",
+                "action: \"DELETE\""
             }
+        ),
+        @Example(
+            title = "Download files from an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_downloads
+              namespace: company.team
+              tasks:
+                - id: "downloads"
+                  type: "io.kestra.plugin.minio.Downloads"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com
+                  bucket: "kestra-test-bucket"
+                  prefix: "data/orders"
+                  action: "DELETE"
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/List.java b/src/main/java/io/kestra/plugin/minio/List.java
@@ -36,6 +36,21 @@
                 "bucket: \"my-bucket\"",
                 "prefix: \"sub-dir\""
             }
+        ),
+        @Example(
+            title = "List files from an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_list
+              namespace: company.team
+              tasks:
+                - id: "list_objects"
+                  type: "io.kestra.plugin.minio.List"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com
+                  bucket: "kestra-test-bucket"
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/Trigger.java b/src/main/java/io/kestra/plugin/minio/Trigger.java
@@ -97,6 +97,35 @@
                 "    prefix: \"sub-dir\"",
                 "    action: NONE",
             }
+        ),
+        @Example(
+            title = "Wait for a list of files on a bucket on an S3-compatible storage — here, Spaces Object Storage from Digital Ocean. Iterate through those files, and move it to another folder.",
+            full = true,
+            code = """
+              id: trigger_on_s3_compatible_storage
+              namespace: company.team
+              tasks:
+                - id: each
+                  type: io.kestra.plugin.core.flow.EachSequential
+                  tasks:
+                    - id: return
+                      type: io.kestra.plugin.core.debug.Return
+                      format: "{{ taskrun.value }}"
+                  value: "{{ trigger.objects | jq('.[].uri') }}"
+              
+              triggers:
+                - id: watch
+                  type: io.kestra.plugin.minio.Trigger
+                  interval: "PT5M"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>>.digitaloceanspaces.com
+                  bucket: "kestra-test-bucket"
+                  prefix: "sub-dir"
+                  action: MOVE
+                  moveTo: 
+                    key: archive
+              """
         )
     }
 )

diff --git a/src/main/java/io/kestra/plugin/minio/Upload.java b/src/main/java/io/kestra/plugin/minio/Upload.java
@@ -46,6 +46,26 @@
                 "bucket: \"my-bucket\"",
                 "key: \"path/to/file\""
             }
+        ),
+        @Example(
+            title = "Upload file to an S3-compatible storage — here, Spaces Object Storage from Digital Ocean.",
+            full = true,
+            code = """
+              id: s3_compatible_upload
+              namespace: company.team
+              tasks:
+                - id: http_download
+                  type: io.kestra.plugin.core.http.Download
+                  uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv
+                - id: "upload_to_storage"
+                  type: "io.kestra.plugin.minio.Upload"
+                  accessKeyId: "<access-key>"
+                  secretKeyId: "<secret-key>"
+                  endpoint: https://<region>.digitaloceanspaces.com  #example regions: nyc3, tor1
+                  bucket: "kestra-test-bucket"
+                  from: "{{ outputs.http_download.uri }}"
+                  key: "data/orders.csv"
+              """
         )
     }
 )