Implement MinIO plugin

build.gradle

@@ -26,7 +26,7 @@ sourceCompatibility = 21
 targetCompatibility = 21
 
 group "io.kestra.plugin"
-description 'Plugin template for Kestra'
+description 'Plugin MinIO for Kestra'
 
 tasks.withType(JavaCompile) {
     options.encoding = "UTF-8"
@@ -48,6 +48,9 @@ dependencies {
 
     // libs included in the final jar
     api group: 'com.google.code.gson', name: 'gson', version: '2.11.0'
+
+    // Minio client dependency
+    implementation 'io.minio:minio:8.5.10'
 }
 
 
@@ -67,6 +70,15 @@ testlogger {
     showSkippedStandardStreams true
 }
 
+// Due to bug that's cannot see com.fasterxml.jackson.annotation.JsonKey which is required in tests
+configurations.all {
+    resolutionStrategy.eachDependency { DependencyResolveDetails details ->
+        if (details.requested.group == 'com.fasterxml.jackson.core' && details.requested.name.startsWith('jackson-')) {
+            details.useVersion '2.17.1'
+        }
+    }
+}
+
 dependencies {
     // lombok
     testAnnotationProcessor "org.projectlombok:lombok:" + lombokVersion
@@ -84,18 +96,32 @@ dependencies {
     // Kestra
     testAnnotationProcessor group: "io.kestra", name: "processor", version: kestraVersion
 
+    testAnnotationProcessor enforcedPlatform("io.kestra:platform:$kestraVersion")
+    testImplementation enforcedPlatform("io.kestra:platform:$kestraVersion")
+
     // test deps needed only for to have a runner
     testAnnotationProcessor group: "io.kestra", name: "processor", version: kestraVersion
     testImplementation group: "io.kestra", name: "core", version: kestraVersion
-    testImplementation group: "io.kestra", name: "core", version: kestraVersion, classifier: "tests"
     testImplementation group: "io.kestra", name: "repository-memory", version: kestraVersion
     testImplementation group: "io.kestra", name: "runner-memory", version: kestraVersion
     testImplementation group: "io.kestra", name: "storage-local", version: kestraVersion
+    testImplementation group: "io.kestra", name: "tests", version: kestraVersion
 
     // test
     testImplementation "org.junit.jupiter:junit-jupiter-engine"
-    testImplementation "org.hamcrest:hamcrest:2.2"
-    testImplementation "org.hamcrest:hamcrest-library:2.2"
+    testImplementation "org.hamcrest:hamcrest"
+    testImplementation "org.hamcrest:hamcrest-library"
+    testImplementation 'org.mockito:mockito-junit-jupiter'
+
+    // testcontainers
+    testImplementation "org.testcontainers:testcontainers:1.19.8"
+    testImplementation "org.testcontainers:junit-jupiter:1.19.8"
+    testImplementation "org.testcontainers:minio:1.19.8"
+    testImplementation "org.testcontainers:localstack:1.19.8"
+
+    testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.17.1'
+    testImplementation 'com.fasterxml.jackson.core:jackson-annotations:2.17.1'
+    testImplementation 'com.fasterxml.jackson.core:jackson-core:2.17.1'
 }
 
 /**********************************************************************************************************************\
@@ -150,8 +176,8 @@ jar {
     manifest {
         attributes(
                 "X-Kestra-Name": project.name,
-                "X-Kestra-Title": "Template",
-                "X-Kestra-Group": project.group + ".templates",
+                "X-Kestra-Title": "MinIO",
+                "X-Kestra-Group": project.group + ".minio",
                 "X-Kestra-Description": project.description,
                 "X-Kestra-Version": project.version
         )

settings.gradle

@@ -1 +1 @@
-rootProject.name = 'plugin-template'
+rootProject.name = 'plugin-minio'

src/main/java/io/kestra/plugin/minio/AbstractMinio.java

@@ -0,0 +1,53 @@
+package io.kestra.plugin.minio;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.runners.RunContext;
+import io.minio.MinioAsyncClient;
+import io.minio.MinioClient;
+import org.apache.commons.lang3.StringUtils;
+
+public interface AbstractMinio extends MinioConnectionInterface {
+
+    default MinioClient client(final RunContext runContext) throws IllegalVariableEvaluationException {
+        MinioConnection.MinioClientConfig minioClientConfig = minioClientConfig(runContext);
+
+        MinioClient.Builder clientBuilder = MinioClient.builder();
+
+        if (StringUtils.isNotEmpty(minioClientConfig.accessKeyId()) &&
+            StringUtils.isNotEmpty(minioClientConfig.secretKeyId())) {
+            clientBuilder.credentials(minioClientConfig.accessKeyId(), minioClientConfig.secretKeyId());
+        }
+
+        if (StringUtils.isNotEmpty(minioClientConfig.endpoint())) {
+            clientBuilder.endpoint(minioClientConfig.endpoint());
+        }
+
+        if (StringUtils.isNotEmpty(minioClientConfig.region())) {
+            clientBuilder.region(minioClientConfig.region());
+        }
+
+        return clientBuilder.build();
+    }
+
+    default MinioAsyncClient asyncClient(final RunContext runContext) throws IllegalVariableEvaluationException {
+        MinioConnection.MinioClientConfig minioClientConfig = minioClientConfig(runContext);
+
+        MinioAsyncClient.Builder clientBuilder = MinioAsyncClient.builder();
+
+        if (StringUtils.isNotEmpty(minioClientConfig.accessKeyId()) &&
+            StringUtils.isNotEmpty(minioClientConfig.secretKeyId())) {
+            clientBuilder.credentials(minioClientConfig.accessKeyId(), minioClientConfig.secretKeyId());
+        }
+
+        if (StringUtils.isNotEmpty(minioClientConfig.endpoint())) {
+            clientBuilder.endpoint(minioClientConfig.endpoint());
+        }
+
+        if (StringUtils.isNotEmpty(minioClientConfig.region())) {
+            clientBuilder.region(minioClientConfig.region());
+        }
+
+        return clientBuilder.build();
+    }
+
+}

src/main/java/io/kestra/plugin/minio/AbstractMinioObject.java

@@ -0,0 +1,24 @@
+package io.kestra.plugin.minio;
+
+import io.kestra.core.models.annotations.PluginProperty;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import lombok.experimental.SuperBuilder;
+
+@SuperBuilder
+@ToString
+@EqualsAndHashCode
+@Getter
+@NoArgsConstructor
+public abstract class AbstractMinioObject extends MinioConnection implements AbstractMinio {
+
+    @Schema(
+        title = "The S3 bucket name."
+    )
+    @PluginProperty(dynamic = true)
+    protected String bucket;
+
+}

src/main/java/io/kestra/plugin/minio/Copy.java

@@ -0,0 +1,142 @@
+package io.kestra.plugin.minio;
+
+import io.kestra.core.models.annotations.Example;
+import io.kestra.core.models.annotations.Plugin;
+import io.kestra.core.models.annotations.PluginProperty;
+import io.kestra.core.models.tasks.RunnableTask;
+import io.kestra.core.runners.RunContext;
+import io.kestra.plugin.minio.model.ObjectOutput;
+import io.minio.CopyObjectArgs;
+import io.minio.CopySource;
+import io.minio.MinioClient;
+import io.minio.ObjectWriteResponse;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.*;
+import lombok.experimental.SuperBuilder;
+
+@SuperBuilder
+@ToString
+@EqualsAndHashCode
+@Getter
+@NoArgsConstructor
+@Plugin(
+    examples = {
+        @Example(
+            code = {
+                "accessKeyId: \"<access-key>\"",
+                "secretKeyId: \"<secret-key>\"",
+                "region: \"eu-central-1\"",
+                "from:",
+                "  bucket: \"my-bucket\"",
+                "  key: \"path/to/file\"",
+                "to:",
+                "  bucket: \"my-bucket2\"",
+                "  key: \"path/to/file2\"",
+            }
+        )
+    }
+)
+@Schema(
+    title = "Copy a file between buckets."
+)
+public class Copy extends AbstractMinioObject implements RunnableTask<Copy.Output> {
+
+    @Schema(
+        title = "The source bucket and key."
+    )
+    @PluginProperty
+    private CopyObjectFrom from;
+
+    @Schema(
+        title = "The destination bucket and key."
+    )
+    @PluginProperty
+    private CopyObject to;
+
+    @Schema(
+        title = "Whether to delete the source file after download."
+    )
+    @PluginProperty
+    @Builder.Default
+    private Boolean delete = false;
+
+    @Override
+    public Output run(RunContext runContext) throws Exception {
+        try (MinioClient minioClient = this.client(runContext)) {
+            CopySource.Builder sourceBuilder = CopySource.builder()
+                .bucket(runContext.render(this.from.bucket))
+                .object(runContext.render(this.from.key));
+
+            if (this.from.versionId != null) {
+                sourceBuilder.versionId(runContext.render(this.from.versionId));
+            }
+
+            CopyObjectArgs.Builder builder = CopyObjectArgs.builder()
+                .bucket(runContext.render(this.to.bucket != null ? this.to.bucket : this.from.bucket))
+                .object(runContext.render(this.to.key))
+                .source(sourceBuilder.build());
+
+            CopyObjectArgs request = builder.build();
+
+            ObjectWriteResponse response = minioClient.copyObject(request);
+
+            if (this.delete) {
+                Delete.builder()
+                    .id(this.id)
+                    .type(Delete.class.getName())
+                    .region(this.region)
+                    .endpoint(this.endpoint)
+                    .accessKeyId(this.accessKeyId)
+                    .secretKeyId(this.secretKeyId)
+                    .bucket(this.from.bucket)
+                    .key(this.from.key)
+                    .build()
+                    .run(runContext);
+            }
+
+            return Output
+                .builder()
+                .bucket(response.bucket())
+                .key(response.object())
+                .eTag(response.etag())
+                .build();
+        }
+    }
+
+    @SuperBuilder(toBuilder = true)
+    @Getter
+    @NoArgsConstructor
+    public static class CopyObject {
+        @Schema(
+            title = "The bucket name"
+        )
+        @PluginProperty(dynamic = true)
+        String bucket;
+
+        @Schema(
+            title = "The bucket key"
+        )
+        @PluginProperty(dynamic = true)
+        String key;
+    }
+
+    @SuperBuilder(toBuilder = true)
+    @Getter
+    @NoArgsConstructor
+    public static class CopyObjectFrom extends CopyObject {
+        @Schema(
+            title = "The specific version of the object."
+        )
+        @PluginProperty(dynamic = true)
+        private String versionId;
+    }
+
+    @SuperBuilder
+    @Getter
+    @NoArgsConstructor
+    public static class Output extends ObjectOutput implements io.kestra.core.models.tasks.Output {
+        private String bucket;
+        private String key;
+    }
+
+}

src/main/java/io/kestra/plugin/minio/CreateBucket.java

@@ -0,0 +1,79 @@
+package io.kestra.plugin.minio;
+
+import io.kestra.core.models.annotations.Example;
+import io.kestra.core.models.annotations.Plugin;
+import io.kestra.core.models.annotations.PluginProperty;
+import io.kestra.core.models.tasks.RunnableTask;
+import io.kestra.core.runners.RunContext;
+import io.minio.BucketExistsArgs;
+import io.minio.MakeBucketArgs;
+import io.minio.MinioClient;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.*;
+import lombok.experimental.SuperBuilder;
+
+@SuperBuilder
+@ToString
+@EqualsAndHashCode
+@Getter
+@NoArgsConstructor
+@Plugin(
+    examples = {
+        @Example(
+            title = "Create a new bucket with some options",
+            code = {
+                "accessKeyId: \"<access-key>\"",
+                "secretKeyId: \"<secret-key>\"",
+                "region: \"eu-central-1\"",
+                "bucket: \"my-bucket\""
+            }
+        )
+    }
+)
+@Schema(
+    title = "Create a bucket"
+)
+public class CreateBucket extends AbstractMinioObject implements RunnableTask<CreateBucket.Output> {
+
+    @Schema(
+        title = "Specifies whether you want Object Lock to be enabled for the new bucket."
+    )
+    @PluginProperty
+    private Boolean objectLockEnabledForBucket;
+
+    @Override
+    public Output run(RunContext runContext) throws Exception {
+        String bucket = runContext.render(this.bucket);
+
+        try (MinioClient client = this.client(runContext)) {
+
+            BucketExistsArgs bucketExistsArgs = BucketExistsArgs.builder().bucket(bucket).build();
+
+            if (client.bucketExists(bucketExistsArgs)) {
+                return Output
+                    .builder()
+                    .bucket(bucket)
+                    .build();
+            }
+            MakeBucketArgs.Builder requestBuilder = MakeBucketArgs.builder().bucket(bucket);
+
+            if (this.objectLockEnabledForBucket != null) {
+                requestBuilder.objectLock(objectLockEnabledForBucket);
+            }
+
+            client.makeBucket(requestBuilder.build());
+
+            return Output
+                .builder()
+                .bucket(bucket)
+                .build();
+        }
+    }
+
+    @Builder
+    @Getter
+    public static class Output implements io.kestra.core.models.tasks.Output {
+        private final String bucket;
+    }
+
+}