diff --git a/.github/workflows/test.storage-s3-compatible.yml b/.github/workflows/test.storage-s3-compatible.yml new file mode 100644 index 00000000..1872cc95 --- /dev/null +++ b/.github/workflows/test.storage-s3-compatible.yml @@ -0,0 +1,56 @@ +name: S3 Compatible +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + + +jobs: + + test: + name: Test + env: + _access_key: minioadmin + _secret_key: minioadmin + _bucket: test-bucket + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [22.x] + steps: + - name: Setup minio + run: >- + docker run -d -p 9000:9000 --name minio + -e MINIO_ACCESS_KEY=${{env._access_key}} + -e MINIO_SECRET_KEY=${{env._secret_key}} + -v /tmp/data:/data -v /tmp/config:/root/.minio + minio/minio server /data + - run: wget https://dl.min.io/client/mc/release/linux-amd64/mc + - run: chmod +x ./mc + - run: ./mc alias set myminio http://127.0.0.1:9000 ${{env._access_key}} ${{env._secret_key}} + - run: ./mc mb --ignore-existing myminio/${{env._bucket}} + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - name: Install dependencies + working-directory: ./packages/storage/storage-s3-compatible + run: npm ci + + - name: Test + run: npm test + working-directory: ./packages/storage/storage-s3-compatible + env: + ACCESS_KEY_ID: ${{env._access_key}} + SECRET_ACCESS_KEY: ${{env._secret_key}} + BUCKET: ${{env._bucket}} + ENDPOINT: http://127.0.0.1:9000 + REGION: us-east-1 + \ No newline at end of file diff --git a/README.md b/README.md index 284dcc57..d1b25633 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ [![Core](https://github.com/store-craft/storecraft/actions/workflows/test.core.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.core.yml) [![MongoDB](https://github.com/store-craft/storecraft/actions/workflows/test.database-mongodb.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.database-mongodb.yml)[![SQLite / Postgres / MySQL](https://github.com/store-craft/storecraft/actions/workflows/test.database-sql.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.database-sql.yml) +[![S3 Compatible](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml) + # The mono-repo diff --git a/package-lock.json b/package-lock.json index db11aa39..41d6e1e1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20159,9 +20159,9 @@ } }, "packages/cli/node_modules/@storecraft/storage-s3-compatible": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@storecraft/storage-s3-compatible/-/storage-s3-compatible-1.0.6.tgz", - "integrity": "sha512-9uxvMDwaEWeW7BHqrpeIICp8G7J+w8Kdw0CjgNLzTxyuhZ+Y9TU04TbiKoODB7qBDpp9AoXzPTVVHnVF0gFB+w==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@storecraft/storage-s3-compatible/-/storage-s3-compatible-1.0.7.tgz", + "integrity": "sha512-L0S+7r4AcqLurrh/57qtxC/LoQN2A/T615KHHyEpv6vaC7L3IbL/TlrW9hJZ2tXCRpvrrvR2jnMdbA+VhdvO3w==", "dev": true, "license": "MIT", "dependencies": { @@ -20579,7 +20579,7 @@ }, "packages/storage/storage-s3-compatible": { "name": "@storecraft/storage-s3-compatible", - "version": "1.0.7", + "version": "1.0.8", "license": "MIT", "dependencies": { "@storecraft/core": "^1.0.0" diff --git a/packages/core/rest/con.storage.routes.js b/packages/core/rest/con.storage.routes.js index a5c15d17..71572921 100644 --- a/packages/core/rest/con.storage.routes.js +++ b/packages/core/rest/con.storage.routes.js @@ -56,7 +56,11 @@ export const create_routes = (app) => { res.headers.set(HEADER_PRESIGNED, 'true'); res.sendJson(r); } else { - await app.storage.putStream(file_key, req.body); + + await app.storage.putStream( + file_key, req.body, {}, + parseInt(req.headers.get("Content-Length") ?? '0') + ); res.headers.set(HEADER_PRESIGNED, 'false'); res.end(); diff --git a/packages/core/storage/types.storage.d.ts b/packages/core/storage/types.storage.d.ts index a486e5e5..98c1d27c 100644 --- a/packages/core/storage/types.storage.d.ts +++ b/packages/core/storage/types.storage.d.ts @@ -85,7 +85,7 @@ export declare interface storage_driver { */ putBlob: (key: string, blob: Blob, meta?: MetaData) => Promise; putArraybuffer: (key: string, buffer: ArrayBuffer, meta?: MetaData) => Promise; - putStream: (key: string, stream: Partial>, meta?: MetaData) => Promise; + putStream: (key: string, stream: Partial>, meta?: MetaData, bytesLength: number) => Promise; putSigned?: (key: string) => Promise; /** diff --git a/packages/core/test-runner/storage/index.js b/packages/core/test-runner/storage/index.js index 6083d465..a93873c1 100644 --- a/packages/core/test-runner/storage/index.js +++ b/packages/core/test-runner/storage/index.js @@ -25,6 +25,14 @@ const sleep = (ms=1000) => new Promise( } ) +function buffer_to_arraybuffer(buffer) { + const arrayBuffer = new ArrayBuffer(buffer.length); + const view = new Uint8Array(arrayBuffer); + for (let i = 0; i < buffer.length; ++i) { + view[i] = buffer[i]; + } + return arrayBuffer; +} /** * * @param {ReadableStream} stream @@ -54,8 +62,8 @@ const readableStreamToArrayBuffer = async (stream) => { */ const areStreamsEqual = async (lhs, rhs) => { return areArrayBuffersEqual( - await readableStreamToArrayBuffer(lhs), - await readableStreamToArrayBuffer(rhs) + (await readableStreamToArrayBuffer(lhs)).buffer, + (await readableStreamToArrayBuffer(rhs)).buffer ); } /** @@ -94,19 +102,24 @@ export const create = (storage, name) => { const data = data_with_buffers; for (const d of data) { + + const as_array_buffer = buffer_to_arraybuffer(d.buffer); - await storage.putArraybuffer(d.key, d.buffer); + await storage.putArraybuffer(d.key, as_array_buffer); // read const { value } = await storage.getArraybuffer(d.key); + + // console.log('as_array_buffer', as_array_buffer) + // console.log('value', value) + // console.log('decoded', new TextDecoder("utf-8").decode(value)) // compare - const equal = areArrayBuffersEqual(d.buffer, value); + const equal = areArrayBuffersEqual(as_array_buffer, value); assert.ok(equal, 'are not equal !!!'); } }); - s('BLOB put/get', async () => { const data = data_with_buffers.map( @@ -134,18 +147,24 @@ export const create = (storage, name) => { ...d, stream: Readable.toWeb(Readable.from(d.buffer)), // stream: Readable.toWeb(createReadStream('node.png')), - key: 'folder1/stream_node.png' + key: 'folder1/stream_node.png', + length: d.buffer.byteLength }) ); for (const d of data) { // @ts-ignore - await storage.putStream(d.key, d.stream); + const success = await storage.putStream( + d.key, d.stream, {}, d.buffer.byteLength + ); // read - const { value } = await storage.getStream(d.key); + const get_stream = await storage.getStream(d.key); + + console.log('success ', success); + console.log('get_stream ', get_stream); // let's read - const reader = value.getReader(); + const reader = get_stream.value.getReader(); let stream_bytes_length = 0; while(true) { const {done, value: chunk } = await reader.read(); @@ -172,15 +191,15 @@ export const create = (storage, name) => { const key = 'folder-test/about_to_be_removed.png' const buffer = data_with_buffers[0].buffer; - await storage.putArraybuffer(key, buffer); + await storage.putArraybuffer(key, buffer_to_arraybuffer(buffer)); // await sleep(1000); await storage.remove(key); - // await sleep(1000); + await sleep(2000); const removed = await storage.getArraybuffer(key); assert.ok( - (removed.value===undefined) || - (removed.value.byteLength==0), + (removed.error) || + (!Boolean(removed.value)), 'not removed !!!' ); }); diff --git a/packages/storage/storage-s3-compatible/README.md b/packages/storage/storage-s3-compatible/README.md index 5db3c97c..32104a84 100644 --- a/packages/storage/storage-s3-compatible/README.md +++ b/packages/storage/storage-s3-compatible/README.md @@ -5,6 +5,8 @@ width='90%' />

+[![S3 Compatible](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml) + `fetch` ready support for an `S3` like storage: - `Amazon S3` - `Cloudflare R2` diff --git a/packages/storage/storage-s3-compatible/adapter.js b/packages/storage/storage-s3-compatible/adapter.js index e2652589..8589da0b 100644 --- a/packages/storage/storage-s3-compatible/adapter.js +++ b/packages/storage/storage-s3-compatible/adapter.js @@ -95,23 +95,30 @@ export class S3CompatibleStorage { * * @param {string} key * @param {BodyInit} body + * @param {object} [headers={}] */ - async #put_internal(key, body) { + async #put_internal(key, body, headers={}) { const r = await this.client.fetch( this.get_file_url(key), { method: 'PUT', - body + body, + headers } ); + if(!r.ok) { + console.log( + await r.text() + ); + } + return r.ok; } /** * - * @param {string} key - * @param {Blob} blob + * @type {storage["putBlob"]} */ async putBlob(key, blob) { return this.#put_internal(key, blob); @@ -119,8 +126,7 @@ export class S3CompatibleStorage { /** * - * @param {string} key - * @param {ArrayBuffer} buffer + * @type {storage["putArraybuffer"]} */ async putArraybuffer(key, buffer) { return this.#put_internal(key, buffer); @@ -128,17 +134,23 @@ export class S3CompatibleStorage { /** * - * @param {string} key - * @param {ReadableStream} stream + * @type {storage["putStream"]} */ - async putStream(key, stream) { - return this.#put_internal(key, stream); + async putStream(key, stream, meta={}, bytesLength=0) { + const extra_headers = {}; + if(Boolean(bytesLength)) { + extra_headers["Content-Length"] = bytesLength; + } + + return this.#put_internal( + // @ts-ignore + key, stream, extra_headers + ); } /** * - * @param {string} key - * @returns {ReturnType} + * @type {storage["putSigned"]} */ async putSigned(key) { const url = new URL(this.get_file_url(key)); @@ -171,14 +183,14 @@ export class S3CompatibleStorage { } /** - * - * @param {string} key + * @type {storage["getArraybuffer"]} */ async getArraybuffer(key) { const r = await this.#get_request(key); - const b = await r.arrayBuffer(); return { - value: b, + value: r.ok ? (await r.arrayBuffer()) : undefined, + error: !r.ok, + message: r.ok ? undefined : await r.text(), metadata: { contentType: infer_content_type(key) } @@ -187,13 +199,14 @@ export class S3CompatibleStorage { /** * - * @param {string} key + * @type {storage["getBlob"]} */ async getBlob(key) { const r = await this.#get_request(key); - const b = await r.blob(); return { - value: b, + value: r.ok ? (await r.blob()) : undefined, + error: !r.ok, + message: r.ok ? undefined : await r.text(), metadata: { contentType: infer_content_type(key) } @@ -201,15 +214,15 @@ export class S3CompatibleStorage { } /** - * - * @param {string} key - * @param {Response} key + * @type {storage["getStream"]} */ async getStream(key) { - - const s = (await this.#get_request(key)).body + const r = await this.#get_request(key); + const b = r.body; return { - value: s, + value: r.ok ? b : undefined, + error: !r.ok, + message: r.ok ? undefined : await r.text(), metadata: { contentType: infer_content_type(key) } @@ -218,8 +231,7 @@ export class S3CompatibleStorage { /** * - * @param {string} key - * @returns {ReturnType} + * @type {storage["getSigned"]} */ async getSigned(key) { const url = new URL(this.get_file_url(key)); @@ -244,7 +256,7 @@ export class S3CompatibleStorage { /** * - * @param {string} key + * @type {storage["remove"]} */ async remove(key) { const r = await this.client.fetch( diff --git a/packages/storage/storage-s3-compatible/package.json b/packages/storage/storage-s3-compatible/package.json index ef80f98d..c7597ff3 100644 --- a/packages/storage/storage-s3-compatible/package.json +++ b/packages/storage/storage-s3-compatible/package.json @@ -1,6 +1,6 @@ { "name": "@storecraft/storage-s3-compatible", - "version": "1.0.7", + "version": "1.0.8", "description": "Official S3-Compatible Storage adapter for storecraft", "license": "MIT", "author": "Tomer Shalev (https://github.com/store-craft)", @@ -17,9 +17,9 @@ "storecraft" ], "scripts": { - "storage-s3-compatible:test": "uvu -c", - "test": "npm run storage-s3-compatible:test", - "prepublishOnly": "npm version patch --force" + "test": "node ./tests/storage.s3-compatible.test.js", + "prepublishOnly": "npm version patch --force", + "sc-publish": "npm publish" }, "type": "module", "main": "adapter.js", diff --git a/packages/storage/storage-s3-compatible/tests/storage.s3-compatible.test.js b/packages/storage/storage-s3-compatible/tests/storage.s3-compatible.test.js new file mode 100644 index 00000000..8728cb1f --- /dev/null +++ b/packages/storage/storage-s3-compatible/tests/storage.s3-compatible.test.js @@ -0,0 +1,22 @@ +import 'dotenv/config'; +import { S3CompatibleStorage } from '@storecraft/storage-s3-compatible' +import { storage as storage_test_runner } from '@storecraft/core/test-runner' + +const FORCE_PATH_STYLE = true; + +const storage = new S3CompatibleStorage( + { + accessKeyId: process.env.ACCESS_KEY_ID, + secretAccessKey: process.env.SECRET_ACCESS_KEY, + bucket: process.env.BUCKET, + endpoint: process.env.ENDPOINT, + forcePathStyle: FORCE_PATH_STYLE, + region: process.env.REGION + } +); + +const suite = storage_test_runner.create(storage); + +suite.before(async () => { await storage.init(undefined) }); + +suite.run();