From e67a355b13b5eb3de08e0cf6719fa612c4dd5d28 Mon Sep 17 00:00:00 2001 From: Oleg Yakovlev Date: Thu, 25 Jul 2024 11:31:40 +0700 Subject: [PATCH] added correct method to support putObject via inputStream --- README.md | 2 +- storage-service/build.gradle.kts | 2 +- .../service/storage/preview/PreviewService.kt | 8 +++-- .../service/storage/s3/IS3Storage.kt | 11 ++++-- .../service/storage/s3/S3StorageImpl.kt | 34 ++++++++++++++++++- .../service/storage/s3/dto/FileObjectDto.kt | 8 +++++ .../src/test/kotlin/S3StorageTest.kt | 9 +++-- 7 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/dto/FileObjectDto.kt diff --git a/README.md b/README.md index a4852fe..eeb8473 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ repositories { } // Append dependency -implementation("com.icerockdev:storage-service:0.7.0") +implementation("com.icerockdev:storage-service:0.8.0") ```` ## Library usage diff --git a/storage-service/build.gradle.kts b/storage-service/build.gradle.kts index fde5139..e1c5637 100644 --- a/storage-service/build.gradle.kts +++ b/storage-service/build.gradle.kts @@ -15,7 +15,7 @@ apply(plugin = "java") apply(plugin = "kotlin") group = "com.icerockdev" -version = "0.7.0" +version = "0.8.0" val sourcesJar by tasks.registering(Jar::class) { archiveClassifier.set("sources") diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/preview/PreviewService.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/preview/PreviewService.kt index eb387eb..65a782f 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/preview/PreviewService.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/preview/PreviewService.kt @@ -5,11 +5,11 @@ package com.icerockdev.service.storage.preview import com.icerockdev.service.storage.s3.IS3Storage +import com.icerockdev.service.storage.s3.dto.FileObjectDto import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.catch -import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.withContext import org.slf4j.LoggerFactory @@ -69,7 +69,11 @@ class PreviewService( val preview = it.imageProcessor(it, imageBytes) val dstKey = getPreviewName(srcKey, it) - storage.put(dstBucket, dstKey, preview) + storage.put( + dstBucket, + dstKey, + FileObjectDto(preview.inputStream().buffered(), preview.size.toLong()) + ) } } catch (e: PreviewException){ logger.error(e.localizedMessage, e) diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt index 24ccdeb..e7b4951 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/IS3Storage.kt @@ -4,12 +4,12 @@ package com.icerockdev.service.storage.s3 +import com.icerockdev.service.storage.s3.dto.FileObjectDto import com.icerockdev.service.storage.s3.policy.builder.PolicyBuilder import com.icerockdev.service.storage.s3.policy.builder.PrincipalBuilder import com.icerockdev.service.storage.s3.policy.builder.ResourceBuilder import com.icerockdev.service.storage.s3.policy.builder.StatementBuilder import com.icerockdev.service.storage.s3.policy.dto.Principal -import com.icerockdev.service.storage.s3.policy.dto.Resource import com.icerockdev.service.storage.s3.policy.dto.Statement import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.services.s3.S3Configuration @@ -44,8 +44,11 @@ interface IS3Storage { fun put(bucket: String, key: String, stream: InputStream, metadata: Map? = null): Boolean + @Deprecated("Pointless method that simply wraps byteArray") fun put(bucket: String, key: String, byteArray: ByteArray, metadata: Map? = null): Boolean + fun put(bucket: String, key: String, file: FileObjectDto, metadata: Map? = null): Boolean + fun copy(srcBucket: String, srcKey: String, dstBucket: String, dstKey: String): Boolean /** @@ -59,7 +62,11 @@ interface IS3Storage { fun getBucketPolicy(bucket: String): String? - fun putBucketPolicy(bucket: String, confirmRemoveSelfBucketAccess: Boolean = false, configure: PolicyBuilder.() -> Unit): Boolean + fun putBucketPolicy( + bucket: String, + confirmRemoveSelfBucketAccess: Boolean = false, + configure: PolicyBuilder.() -> Unit + ): Boolean fun deleteBucketPolicy(bucket: String): Boolean diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt index bdd3dc0..70bbc3e 100644 --- a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/S3StorageImpl.kt @@ -5,6 +5,7 @@ package com.icerockdev.service.storage.s3 import com.icerockdev.service.storage.mime.MimeTypeDetector +import com.icerockdev.service.storage.s3.dto.FileObjectDto import com.icerockdev.service.storage.s3.policy.builder.PolicyBuilder import com.icerockdev.service.storage.s3.policy.builder.PrincipalBuilder import com.icerockdev.service.storage.s3.policy.builder.ResourceBuilder @@ -58,6 +59,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi .build() ) } catch (e: NoSuchKeyException) { + logger.debug(e.localizedMessage, e) null } } @@ -125,6 +127,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi client.headBucket(HeadBucketRequest.builder().bucket(bucket).build()) true } catch (e: NoSuchBucketException) { + logger.debug(e.localizedMessage, e) false } } @@ -139,11 +142,13 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi ) true } catch (e: BucketAlreadyExistsException) { + logger.debug(e.localizedMessage, e) false } catch (e: BucketAlreadyOwnedByYouException) { + logger.debug(e.localizedMessage, e) false } catch (e: S3Exception) { - logger.error(e.localizedMessage, e) + logger.debug(e.localizedMessage, e) false } } @@ -153,6 +158,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi client.deleteBucket(DeleteBucketRequest.builder().bucket(bucket).build()) true } catch (e: NoSuchBucketException) { + logger.debug(e.localizedMessage, e) false } } @@ -174,6 +180,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi ) true } catch (e: NoSuchKeyException) { + logger.debug(e.localizedMessage, e) false } } @@ -182,6 +189,7 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi return put(bucket, key, stream.buffered(), metadata) } + @Deprecated("Pointless method that simply wraps byteArray") override fun put(bucket: String, key: String, byteArray: ByteArray, metadata: Map?): Boolean { val stream = byteArray.inputStream().buffered() return put(bucket, key, stream, metadata) @@ -202,6 +210,28 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi client.putObject(request, RequestBody.fromBytes(stream.readBytes())) true } catch (e: S3Exception) { + logger.debug(e.localizedMessage, e) + false + } + } + + override fun put(bucket: String, key: String, file: FileObjectDto, metadata: Map?): Boolean { + + val request = PutObjectRequest.builder() + .bucket(bucket) + .key(key) + .acl(ObjectCannedACL.PUBLIC_READ) + .contentEncoding("UTF-8") + .contentType(MimeTypeDetector.detect(file.inputStream).toString()) + .metadata(metadata ?: emptyMap()) + .contentLength(file.size) + .build() + + return try { + client.putObject(request, RequestBody.fromInputStream(file.inputStream, file.size)) + true + } catch (e: S3Exception) { + logger.debug(e.localizedMessage, e) false } } @@ -219,8 +249,10 @@ class S3StorageImpl(private val client: S3Client, private val preSigner: S3Presi client.copyObject(request) true } catch (e: NoSuchBucketException) { + logger.debug(e.localizedMessage, e) false } catch (e: NoSuchKeyException) { + logger.debug(e.localizedMessage, e) false } } diff --git a/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/dto/FileObjectDto.kt b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/dto/FileObjectDto.kt new file mode 100644 index 0000000..8f1a372 --- /dev/null +++ b/storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/dto/FileObjectDto.kt @@ -0,0 +1,8 @@ +package com.icerockdev.service.storage.s3.dto + +import java.io.BufferedInputStream + +data class FileObjectDto( + val inputStream: BufferedInputStream, + val size: Long +) diff --git a/storage-service/src/test/kotlin/S3StorageTest.kt b/storage-service/src/test/kotlin/S3StorageTest.kt index 7431f67..9ad801e 100644 --- a/storage-service/src/test/kotlin/S3StorageTest.kt +++ b/storage-service/src/test/kotlin/S3StorageTest.kt @@ -8,6 +8,7 @@ import com.icerockdev.service.storage.mime.MimeTypeDetector import com.icerockdev.service.storage.s3.IS3Storage import com.icerockdev.service.storage.s3.S3StorageImpl import com.icerockdev.service.storage.s3.minioConfBuilder +import com.icerockdev.service.storage.s3.dto.FileObjectDto import com.icerockdev.service.storage.s3.policy.dto.ActionEnum import com.icerockdev.service.storage.s3.policy.dto.EffectEnum import com.icerockdev.service.storage.s3.policy.dto.Policy @@ -253,15 +254,13 @@ class S3StorageTest { val (jpgFileName, jpgStream) = getFile(FileType.JPG) val mimeType = MimeTypeDetector.detect(jpgStream) - val jpgFileByteArray = jpgStream.readAllBytes() - - jpgStream.close() + val fileSize = jpgStream.available().toLong() // Check wrong cases assertFalse(storage.objectExists(bucketName, jpgFileName)) // Put object to storage - assertTrue(storage.put(bucketName, jpgFileName, jpgFileByteArray.inputStream())) + assertTrue(storage.put(bucketName, jpgFileName, FileObjectDto(jpgStream.buffered(), fileSize))) // Check object exist assertTrue(storage.objectExists(bucketName, jpgFileName)) @@ -271,7 +270,7 @@ class S3StorageTest { assertEquals("image/jpeg", mimeType.toString()) assertEquals(mimeType.toString(), jpgObject?.response()?.contentType()) - assertEquals(jpgFileByteArray.size.toLong(), jpgObject?.response()?.contentLength()) + assertEquals(fileSize, jpgObject?.response()?.contentLength()) assertTrue { storage.deleteBucketWithObjects(bucketName)