Merge pull request #83 from store-craft/ci_tests_for_s3

ci/cd: using minio to test s3

.github/workflows/test.storage-s3-compatible.yml

@@ -0,0 +1,56 @@
+name: S3 Compatible
+on: 
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+
+jobs:
+
+  test:
+    name: Test
+    env:
+      _access_key: minioadmin
+      _secret_key: minioadmin
+      _bucket: test-bucket
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [22.x]
+    steps:
+      - name: Setup minio
+        run: >-
+          docker run -d -p 9000:9000 --name minio 
+          -e MINIO_ACCESS_KEY=${{env._access_key}} 
+          -e MINIO_SECRET_KEY=${{env._secret_key}} 
+          -v /tmp/data:/data -v /tmp/config:/root/.minio 
+          minio/minio server /data
+      - run: wget https://dl.min.io/client/mc/release/linux-amd64/mc
+      - run: chmod +x ./mc
+      - run: ./mc alias set myminio http://127.0.0.1:9000 ${{env._access_key}} ${{env._secret_key}}
+      - run: ./mc mb --ignore-existing myminio/${{env._bucket}}
+
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        working-directory: ./packages/storage/storage-s3-compatible
+        run: npm ci
+
+      - name: Test
+        run: npm test
+        working-directory: ./packages/storage/storage-s3-compatible
+        env:
+          ACCESS_KEY_ID: ${{env._access_key}}
+          SECRET_ACCESS_KEY: ${{env._secret_key}}
+          BUCKET: ${{env._bucket}}
+          ENDPOINT: http://127.0.0.1:9000
+          REGION: us-east-1
+

README.md

@@ -8,6 +8,8 @@
 
 [![Core](https://github.com/store-craft/storecraft/actions/workflows/test.core.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.core.yml)
 [![MongoDB](https://github.com/store-craft/storecraft/actions/workflows/test.database-mongodb.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.database-mongodb.yml)[![SQLite / Postgres / MySQL](https://github.com/store-craft/storecraft/actions/workflows/test.database-sql.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.database-sql.yml)
+[![S3 Compatible](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml)
+
 
 # The <img src='https://storecraft.app/storecraft-color.svg' height='24px' style="transform: translateY(4px);" /> mono-repo
 

packages/core/rest/con.storage.routes.js

@@ -56,7 +56,11 @@ export const create_routes = (app) => {
         res.headers.set(HEADER_PRESIGNED, 'true');
         res.sendJson(r);
       } else {
-        await app.storage.putStream(file_key, req.body);
+
+        await app.storage.putStream(
+          file_key, req.body, {}, 
+          parseInt(req.headers.get("Content-Length") ?? '0')
+        );
 
         res.headers.set(HEADER_PRESIGNED, 'false');
         res.end();

packages/core/storage/types.storage.d.ts

@@ -85,7 +85,7 @@ export declare interface storage_driver {
    */
   putBlob: (key: string, blob: Blob, meta?: MetaData) => Promise<boolean>; 
   putArraybuffer: (key: string, buffer: ArrayBuffer, meta?: MetaData) => Promise<boolean>; 
-  putStream: (key: string, stream: Partial<ReadableStream<any>>, meta?: MetaData) => Promise<boolean>; 
+  putStream: (key: string, stream: Partial<ReadableStream<any>>, meta?: MetaData, bytesLength: number) => Promise<boolean>; 
   putSigned?: (key: string) => Promise<StorageSignedOperation | undefined>; 
 
   /**

packages/core/test-runner/storage/index.js

@@ -25,6 +25,14 @@ const sleep = (ms=1000) => new Promise(
   }
 )
 
+function buffer_to_arraybuffer(buffer) {
+  const arrayBuffer = new ArrayBuffer(buffer.length);
+  const view = new Uint8Array(arrayBuffer);
+  for (let i = 0; i < buffer.length; ++i) {
+    view[i] = buffer[i];
+  }
+  return arrayBuffer;
+}
 /**
  * 
  * @param {ReadableStream} stream 
@@ -54,8 +62,8 @@ const readableStreamToArrayBuffer = async (stream) => {
  */
 const areStreamsEqual = async (lhs, rhs) => {
   return areArrayBuffersEqual(
-    await readableStreamToArrayBuffer(lhs), 
-    await readableStreamToArrayBuffer(rhs)
+    (await readableStreamToArrayBuffer(lhs)).buffer, 
+    (await readableStreamToArrayBuffer(rhs)).buffer
   );
 }
 /**
@@ -94,19 +102,24 @@ export const create = (storage, name) => {
     const data = data_with_buffers;
 
     for (const d of data) {
+
+      const as_array_buffer = buffer_to_arraybuffer(d.buffer);
 
-      await storage.putArraybuffer(d.key, d.buffer);
+      await storage.putArraybuffer(d.key, as_array_buffer);
       // read
       const { value } = await storage.getArraybuffer(d.key);
+
+      // console.log('as_array_buffer', as_array_buffer)
+      // console.log('value', value)
+      // console.log('decoded', new TextDecoder("utf-8").decode(value))
       // compare
-      const equal = areArrayBuffersEqual(d.buffer, value);
+      const equal = areArrayBuffersEqual(as_array_buffer, value);
       assert.ok(equal, 'are not equal !!!');
 
     }
 
   });
 
-
   s('BLOB put/get', async () => {
 
     const data = data_with_buffers.map(
@@ -134,18 +147,24 @@ export const create = (storage, name) => {
         ...d,
         stream: Readable.toWeb(Readable.from(d.buffer)),
         // stream: Readable.toWeb(createReadStream('node.png')),
-        key: 'folder1/stream_node.png'
+        key: 'folder1/stream_node.png', 
+        length: d.buffer.byteLength
       })
     );
 
     for (const d of data) {
       // @ts-ignore
-      await storage.putStream(d.key, d.stream);
+      const success = await storage.putStream(
+        d.key, d.stream, {}, d.buffer.byteLength
+      );
       // read
-      const { value } = await storage.getStream(d.key);
+      const get_stream = await storage.getStream(d.key);
+
+      console.log('success ', success);
+      console.log('get_stream ', get_stream);
 
       // let's read
-      const reader = value.getReader();
+      const reader = get_stream.value.getReader();
       let stream_bytes_length = 0;
       while(true) {
         const {done, value: chunk } = await reader.read();
@@ -172,15 +191,15 @@ export const create = (storage, name) => {
     const key = 'folder-test/about_to_be_removed.png'
     const buffer = data_with_buffers[0].buffer;
 
-    await storage.putArraybuffer(key, buffer);
+    await storage.putArraybuffer(key, buffer_to_arraybuffer(buffer));
     // await sleep(1000);
     await storage.remove(key);
-    // await sleep(1000);
+    await sleep(2000);
     const removed = await storage.getArraybuffer(key);
 
     assert.ok(
-      (removed.value===undefined) || 
-      (removed.value.byteLength==0), 
+      (removed.error) || 
+      (!Boolean(removed.value)), 
       'not removed !!!'
     );
   });

packages/storage/storage-s3-compatible/README.md

@@ -5,6 +5,8 @@
        width='90%' />
 </div><hr/><br/>
 
+[![S3 Compatible](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml/badge.svg)](https://github.com/store-craft/storecraft/actions/workflows/test.storage-s3-compatible.yml)
+
 `fetch` ready support for an `S3` like storage:
 - `Amazon S3`
 - `Cloudflare R2`

packages/storage/storage-s3-compatible/adapter.js

@@ -95,50 +95,62 @@ export class S3CompatibleStorage {
    * 
    * @param {string} key 
    * @param {BodyInit} body 
+   * @param {object} [headers={}] 
    */
-  async #put_internal(key, body) {
+  async #put_internal(key, body, headers={}) {
     const r = await this.client.fetch(
       this.get_file_url(key),
       {
         method: 'PUT',
-        body
+        body,
+        headers
       }
     );
 
+    if(!r.ok) {
+      console.log(
+        await r.text()
+      );
+    }
+
     return r.ok;
   }
 
   /**
    * 
-   * @param {string} key 
-   * @param {Blob} blob 
+   * @type {storage["putBlob"]}
    */
   async putBlob(key, blob) {
     return this.#put_internal(key, blob);
   }
 
   /**
    * 
-   * @param {string} key 
-   * @param {ArrayBuffer} buffer 
+   * @type {storage["putArraybuffer"]}
    */
   async putArraybuffer(key, buffer) {
     return this.#put_internal(key, buffer);
   }  
 
   /**
    * 
-   * @param {string} key 
-   * @param {ReadableStream} stream 
+   * @type {storage["putStream"]}
    */
-  async putStream(key, stream) {
-    return this.#put_internal(key, stream);
+  async putStream(key, stream, meta={}, bytesLength=0) {
+    const extra_headers = {};
+    if(Boolean(bytesLength)) {
+      extra_headers["Content-Length"] = bytesLength;
+    }
+
+    return this.#put_internal(
+      // @ts-ignore
+      key, stream, extra_headers
+    );
   }
 
   /**
    * 
-   * @param {string} key 
-   * @returns {ReturnType<import('@storecraft/core/storage').storage_driver["putSigned"]>}
+   * @type {storage["putSigned"]}
    */
   async putSigned(key) {
     const url = new URL(this.get_file_url(key));
@@ -171,14 +183,14 @@ export class S3CompatibleStorage {
   }
 
   /**
-   * 
-   * @param {string} key 
+   * @type {storage["getArraybuffer"]}
    */
   async getArraybuffer(key) {
     const r = await this.#get_request(key);
-    const b = await r.arrayBuffer();
     return {
-      value: b, 
+      value: r.ok ? (await r.arrayBuffer()) : undefined, 
+      error: !r.ok,
+      message: r.ok ? undefined : await r.text(),
       metadata: {
         contentType: infer_content_type(key)
       }
@@ -187,29 +199,30 @@ export class S3CompatibleStorage {
 
   /**
    * 
-   * @param {string} key 
+   * @type {storage["getBlob"]}
    */
   async getBlob(key) {
     const r = await this.#get_request(key);
-    const b = await r.blob();
     return {
-      value: b, 
+      value: r.ok ? (await r.blob()) : undefined, 
+      error: !r.ok,
+      message: r.ok ? undefined : await r.text(),
       metadata: {
         contentType: infer_content_type(key)
       }
     };
   }
 
   /**
-   * 
-   * @param {string} key 
-   * @param {Response} key 
+   * @type {storage["getStream"]}
    */
   async getStream(key) {
-
-    const s = (await this.#get_request(key)).body
+    const r = await this.#get_request(key);
+    const b = r.body;
     return {
-      value: s, 
+      value: r.ok ? b : undefined, 
+      error: !r.ok,
+      message: r.ok ? undefined : await r.text(),
       metadata: {
         contentType: infer_content_type(key)
       }
@@ -218,8 +231,7 @@ export class S3CompatibleStorage {
 
   /**
    * 
-   * @param {string} key 
-   * @returns {ReturnType<import('@storecraft/core/storage').storage_driver["getSigned"]>}
+   * @type {storage["getSigned"]}
    */
   async getSigned(key) {
     const url = new URL(this.get_file_url(key));
@@ -244,7 +256,7 @@ export class S3CompatibleStorage {
 
   /**
    * 
-   * @param {string} key 
+   * @type {storage["remove"]}
    */
   async remove(key) {
     const r = await this.client.fetch(

packages/storage/storage-s3-compatible/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@storecraft/storage-s3-compatible",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "Official S3-Compatible Storage adapter for storecraft",
   "license": "MIT",
   "author": "Tomer Shalev (https://github.com/store-craft)",
@@ -17,9 +17,9 @@
     "storecraft"
   ],
   "scripts": {
-    "storage-s3-compatible:test": "uvu -c",
-    "test": "npm run storage-s3-compatible:test",
-    "prepublishOnly": "npm version patch --force"
+    "test": "node ./tests/storage.s3-compatible.test.js",
+    "prepublishOnly": "npm version patch --force",
+    "sc-publish": "npm publish"
   },
   "type": "module",
   "main": "adapter.js",