From ae24b06379db5425ef7715cc82932058e4f709a8 Mon Sep 17 00:00:00 2001 From: Rafael M Date: Wed, 11 Aug 2021 15:49:46 +0200 Subject: [PATCH 1/3] add optional resumable flag to save function --- src/save.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/save.js b/src/save.js index 4eceedc..80f7eb0 100644 --- a/src/save.js +++ b/src/save.js @@ -25,7 +25,7 @@ const deleteLocalFile = (that, filePath) => }); }); -module.exports = async function (uri, buffer, logPrefix) { +module.exports = async function (uri, buffer, logPrefix, resumable) { try { logPrefix = logPrefix ? [logPrefix, '>'] : []; let structure, bucket, path; @@ -63,12 +63,20 @@ module.exports = async function (uri, buffer, logPrefix) { // log progress this.sdk.log(this, 'log', logPrefix.concat(['storage.save.gs >', uri])); - // upload file to gcs - await this.sdk.gs.bucket(bucket).upload(tempFilePath, { + // create default bucket config + let bucketConfig = { gzip: false, destination: path, metadata: {}, - }); + } + + // update bucket config with resumable flag if set + if (resumable !== undefined && resumable !== null) { + bucketConfig.resumable = resumable + } + + // upload file to gcs + await this.sdk.gs.bucket(bucket).upload(tempFilePath, bucketConfig); // delete local temp file await deleteLocalFile(this, tempFilePath); From db7959dd7dd4397dca079dfa731dadf04c4d067f Mon Sep 17 00:00:00 2001 From: Rafael M Date: Wed, 11 Aug 2021 15:50:27 +0200 Subject: [PATCH 2/3] update dependencies and github actions --- .github/workflows/publish.yml | 4 +- package.json | 6 +-- yarn.lock | 94 +++++++++++++++++++---------------- 3 files changed, 55 insertions(+), 49 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4938bbe..e99127d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,7 +11,7 @@ jobs: - name: 👀 Checkout repo uses: actions/checkout@v2.3.4 - name: 🛠 Setup Node - uses: actions/setup-node@v2.1.5 + uses: actions/setup-node@v2.4.0 with: node-version: '12.x' registry-url: 'https://npm.pkg.github.com' @@ -29,7 +29,7 @@ jobs: - name: 👀 Checkout repo uses: actions/checkout@v2.3.4 - name: 🛠 Setup Node - uses: actions/setup-node@v2.1.5 + uses: actions/setup-node@v2.4.0 with: node-version: '12.x' registry-url: 'https://registry.npmjs.org' diff --git a/package.json b/package.json index a23a451..09af473 100644 --- a/package.json +++ b/package.json @@ -16,14 +16,14 @@ "scripts": { }, "dependencies": { - "@google-cloud/storage": "5.8.5", - "aws-sdk": "2.934.0", + "@google-cloud/storage": "5.13.0", + "aws-sdk": "2.965.0", "node-fetch": "2.6.1", "uuid": "8.3.2" }, "devDependencies": { "@swrlab/swr-prettier-config": "0.1.2", - "eslint": "7.29.0", + "eslint": "7.32.0", "eslint-plugin-swr": "0.0.5" }, "prettier": "@swrlab/swr-prettier-config" diff --git a/yarn.lock b/yarn.lock index 5ab603d..2f1278c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -116,10 +116,10 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" -"@eslint/eslintrc@^0.4.2": - version "0.4.2" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.2.tgz#f63d0ef06f5c0c57d76c4ab5f63d3835c51b0179" - integrity sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg== +"@eslint/eslintrc@^0.4.3": + version "0.4.3" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" + integrity sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw== dependencies: ajv "^6.12.4" debug "^4.1.1" @@ -131,10 +131,10 @@ minimatch "^3.0.4" strip-json-comments "^3.1.1" -"@google-cloud/common@^3.6.0": - version "3.6.0" - resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.6.0.tgz#c2f6da5f79279a4a9ac7c71fc02d582beab98e8b" - integrity sha512-aHIFTqJZmeTNO9md8XxV+ywuvXF3xBm5WNmgWeeCK+XN5X+kGW0WEX94wGwj+/MdOnrVf4dL2RvSIt9J5yJG6Q== +"@google-cloud/common@^3.7.0": + version "3.7.1" + resolved "https://registry.yarnpkg.com/@google-cloud/common/-/common-3.7.1.tgz#e6a4b512ea0c72435b853831565bfba6a8dff2ac" + integrity sha512-BJfcV5BShbunYcn5HniebXLVp2Y6fpuesNegyar5CG8H2AKYHlKxnVID+FSwy92WAW4N2lpGdvxRsmiAn8Fc3w== dependencies: "@google-cloud/projectify" "^2.0.0" "@google-cloud/promisify" "^2.0.0" @@ -143,7 +143,7 @@ ent "^2.2.0" extend "^3.0.2" google-auth-library "^7.0.2" - retry-request "^4.1.1" + retry-request "^4.2.2" teeny-request "^7.0.0" "@google-cloud/paginator@^3.0.0": @@ -164,12 +164,12 @@ resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.2.tgz#81d654b4cb227c65c7ad2f9a7715262febd409ed" integrity sha512-EvuabjzzZ9E2+OaYf+7P9OAiiwbTxKYL0oGLnREQd+Su2NTQBpomkdlkBowFvyWsaV0d1sSGxrKpSNcrhPqbxg== -"@google-cloud/storage@5.8.5": - version "5.8.5" - resolved "https://registry.yarnpkg.com/@google-cloud/storage/-/storage-5.8.5.tgz#2cf1e2e0ef8ca552abc4450301fef3fea4900ef6" - integrity sha512-i0gB9CRwQeOBYP7xuvn14M40LhHCwMjceBjxE4CTvsqL519sVY5yVKxLiAedHWGwUZHJNRa7Q2CmNfkdRwVNPg== +"@google-cloud/storage@5.13.0": + version "5.13.0" + resolved "https://registry.yarnpkg.com/@google-cloud/storage/-/storage-5.13.0.tgz#f40cf085f807c8456acbd994aff01f65845ec9f5" + integrity sha512-LN3Cv7NRBbjwdwcrIFbEaKEEUCUr7aIoBa7j1HXeuBLjC1lQ5VGVGEbaLRP6uwgBKVnf5rFYSTA545C1GR+e/A== dependencies: - "@google-cloud/common" "^3.6.0" + "@google-cloud/common" "^3.7.0" "@google-cloud/paginator" "^3.0.0" "@google-cloud/promisify" "^2.0.0" arrify "^2.0.0" @@ -178,8 +178,7 @@ date-and-time "^1.0.0" duplexify "^4.0.0" extend "^3.0.2" - gaxios "^4.0.0" - gcs-resumable-upload "^3.1.4" + gcs-resumable-upload "^3.3.0" get-stream "^6.0.0" hash-stream-validation "^0.2.2" mime "^2.2.0" @@ -191,6 +190,20 @@ stream-events "^1.0.1" xdg-basedir "^4.0.0" +"@humanwhocodes/config-array@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9" + integrity sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg== + dependencies: + "@humanwhocodes/object-schema" "^1.2.0" + debug "^4.1.1" + minimatch "^3.0.4" + +"@humanwhocodes/object-schema@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.0.tgz#87de7af9c231826fdd68ac7258f77c429e0e5fcf" + integrity sha512-wdppn25U8z/2yiaT6YGquE6X8sSv7hNMWSXYSSU1jGv/yd6XqjXgTDJ8KP4NgjTXfJ3GbRjeeb8RTV7a/VpM+w== + "@swr/prettier-config@0.0.1": version "0.0.1" resolved "https://registry.yarnpkg.com/@swr/prettier-config/-/prettier-config-0.0.1.tgz#11891e5834dc3b5e21c74367b7cbf3caa3a9cbf5" @@ -391,10 +404,10 @@ async-retry@^1.3.1: dependencies: retry "0.12.0" -aws-sdk@2.934.0: - version "2.934.0" - resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.934.0.tgz#1c26bd8ded15d2f6b2aa3301d35fc5daf1a61d68" - integrity sha512-k7p08ewrKcbs0ikCLFi9OI98Iv9dMND5244xPxUIjK5BLtuT/9Gr6eSCHfN70eCQyM5Y2xG1VJP6zhpkihu9Ew== +aws-sdk@2.965.0: + version "2.965.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.965.0.tgz#e63fb38bcc77334a3c9edbe9353050312f3bdbfe" + integrity sha512-jifeFsA6IEKXM65WI5gvBNSCXKw4n64Wf9Q7/8E7wZ5vzRbBGoHzGpyhK6ZBBRvE2YvNp/ykTWChO7RydkA+AQ== dependencies: buffer "4.9.2" events "1.1.1" @@ -872,13 +885,14 @@ eslint-visitor-keys@^2.0.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz#21fdc8fbcd9c795cc0321f0563702095751511a8" integrity sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ== -eslint@7.29.0: - version "7.29.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.29.0.tgz#ee2a7648f2e729485e4d0bd6383ec1deabc8b3c0" - integrity sha512-82G/JToB9qIy/ArBzIWG9xvvwL3R86AlCjtGw+A29OMZDqhTybz/MByORSukGxeI+YPCR4coYyITKk8BFH9nDA== +eslint@7.32.0: + version "7.32.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.32.0.tgz#c6d328a14be3fb08c8d1d21e12c02fdb7a2a812d" + integrity sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA== dependencies: "@babel/code-frame" "7.12.11" - "@eslint/eslintrc" "^0.4.2" + "@eslint/eslintrc" "^0.4.3" + "@humanwhocodes/config-array" "^0.5.0" ajv "^6.10.0" chalk "^4.0.0" cross-spawn "^7.0.2" @@ -1172,10 +1186,10 @@ gcp-metadata@^4.2.0: gaxios "^3.0.0" json-bigint "^1.0.0" -gcs-resumable-upload@^3.1.4: - version "3.2.0" - resolved "https://registry.yarnpkg.com/gcs-resumable-upload/-/gcs-resumable-upload-3.2.0.tgz#9b96b835dcc552523900b60371f3d3bc1f204717" - integrity sha512-k6OAkrO0N1zgvwTRxgCC43K9BMvNUBhZkkFELsMSlgAVs7Sd9C1TA9pmDLBZmFBN8sdvrObsSbCfOOFnHULRvA== +gcs-resumable-upload@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/gcs-resumable-upload/-/gcs-resumable-upload-3.3.0.tgz#d1a866173f9b47e045d4406cafaa658dbb01e624" + integrity sha512-MQKWi+9hOSTyg5/SI1NBW4gAjL1wlkoevHefvr1PCBBXH4uKYLsug5qRrcotWKolDPLfWS51cWaHRN0CTtQNZw== dependencies: abort-controller "^3.0.0" configstore "^5.0.0" @@ -1364,7 +1378,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: +inherits@2, inherits@^2.0.3, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -1970,7 +1984,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -"readable-stream@2 || 3", readable-stream@^3.1.1: +readable-stream@^3.1.1: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -2029,13 +2043,13 @@ ret@~0.1.10: resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== -retry-request@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-4.1.1.tgz#f676d0db0de7a6f122c048626ce7ce12101d2bd8" - integrity sha512-BINDzVtLI2BDukjWmjAIRZ0oglnCAkpP2vQjM3jdLhmT62h0xnQgciPwBRDAvHqpkPT2Wo1XuUyLyn6nbGrZQQ== +retry-request@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/retry-request/-/retry-request-4.2.2.tgz#b7d82210b6d2651ed249ba3497f07ea602f1a903" + integrity sha512-xA93uxUD/rogV7BV59agW/JHPGXeREMWiZc9jhcwY4YdZ7QOtC7qbomYg0n4wyk2lJhggjvKvhNX8wln/Aldhg== dependencies: debug "^4.1.1" - through2 "^3.0.1" + extend "^3.0.2" retry@0.12.0: version "0.12.0" @@ -2330,14 +2344,6 @@ through2@^2.0.0: readable-stream "~2.3.6" xtend "~4.0.1" -through2@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" - integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== - dependencies: - inherits "^2.0.4" - readable-stream "2 || 3" - to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" From 20ae377c8de1935b28f54ad2b3381ea355f12538 Mon Sep 17 00:00:00 2001 From: Rafael M Date: Wed, 11 Aug 2021 15:50:55 +0200 Subject: [PATCH 3/3] update version, readme and changelog --- CHANGELOG.md | 5 +++++ README.md | 4 ++-- package.json | 9 ++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d92524..3b776ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ by [**SWR audio lab**](https://lab.swr.de/) ## Changelog +- 2021-08-11 - v0.2.3 + - updated dependencies + - updated github actions + - add optional 'resumable' flag + - 2021-06-22 - v0.2.2 - updated dependencies - updated github actions diff --git a/README.md b/README.md index 6dbc5bb..8944574 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,11 @@ This module allows easy unified access to the most common storage operations in the local file storage, AWS S3 and Google Cloud Storage. -Please note: This project is still under development! +*Please note: This project is still under development!* ## Installation -t.b.c. +Install dependencies with `yarn`. ## Changelog diff --git a/package.json b/package.json index 09af473..a0f1ac5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@swrlab/node-storage-wrapper", - "version": "0.2.2", + "version": "0.2.3", "description": "Wrapping AWS S3, GCP GCS, file storage", "main": "./src/index.js", "engines": { @@ -8,13 +8,12 @@ }, "repository": { "type": "git", - "url": "https://github.com/swrlab/node-storage-wrapper" + "url": "https://github.com/swrlab/node-storage-wrapper" }, "author": "SWR audio lab ", - "license": "SEE LICENSE in LICENSE.md", + "license": "Hippocratic-2.0", "private": false, - "scripts": { - }, + "scripts": {}, "dependencies": { "@google-cloud/storage": "5.13.0", "aws-sdk": "2.965.0",