updating aws list workflow

.github/workflows/linter.yml

@@ -9,7 +9,7 @@ jobs:
 
     steps:
       - name: Checkout Code
-        uses: actions/[email protected].1
+        uses: actions/[email protected].3
 
       - name: Lint Code Base
         uses: docker://github/super-linter:v2.2.0

.github/workflows/publish.yml

@@ -9,9 +9,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: 👀 Checkout repo
-      uses: actions/[email protected].1
+      uses: actions/[email protected].3
     - name: 🛠 Setup Node
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         node-version: '12.x'
         registry-url: 'https://npm.pkg.github.com'
@@ -27,9 +27,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: 👀 Checkout repo
-      uses: actions/[email protected].1
+      uses: actions/[email protected].3
     - name: 🛠 Setup Node
-      uses: actions/[email protected].1
+      uses: actions/[email protected].2
       with:
         node-version: '12.x'
         registry-url: 'https://registry.npmjs.org'

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@swrlab/node-storage-wrapper",
-	"version": "0.0.12",
+	"version": "0.1.0",
 	"description": "Wrapping AWS S3, GCP GCS, file storage",
 	"main": "./src/index.js",
 	"engines": {
@@ -16,15 +16,15 @@
 	"scripts": {
 	},
 	"dependencies": {
-		"@google-cloud/storage": "5.1.2",
-		"aws-sdk": "2.726.0",
-		"node-fetch": "2.6.0",
-		"uuid": "8.3.0"
+		"@google-cloud/storage": "5.3.0",
+		"aws-sdk": "2.779.0",
+		"node-fetch": "2.6.1",
+		"uuid": "8.3.1"
 	},
 	"devDependencies": {
 		"@swrlab/swr-prettier-config": "0.0.2",
-		"eslint": "^7.6.0",
-		"eslint-plugin-swr": "0.0.3"
+		"eslint": "7.12.1",
+		"eslint-plugin-swr": "0.0.5"
 	},
 	"prettier": "@swrlab/swr-prettier-config"
 }

src/list.js

@@ -11,7 +11,7 @@ const awsListObjects = async (that, bucket, path, next) => {
 			.listObjectsV2({
 				Bucket: bucket,
 				Prefix: path,
-				MaxKeys: 200,
+				MaxKeys: 500,
 				ContinuationToken: next,
 			})
 			.promise();
@@ -22,7 +22,7 @@ const awsListObjects = async (that, bucket, path, next) => {
 			next: files.IsTruncated ? files.NextContinuationToken : null,
 		});
 	} catch (err) {
-		this.error('error', [
+		that.sdk.log('error', [
 			'storage.list.awsListObjects',
 			JSON.stringify({ bucket, path, next, message: err.message, stack: err.stack }),
 		]);
@@ -38,7 +38,7 @@ const listLocalFiles = (that, uri) =>
 		});
 	});
 
-module.exports = async function (uri) {
+module.exports = async function (uri, max, next) {
 	try {
 		let structure, bucket, path, file;
 
@@ -49,25 +49,28 @@ module.exports = async function (uri) {
 			path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.list.aws >', uri]);
+			this.sdk.log(this, 'log', ['storage.list.aws >', uri, max]);
 
 			// load file
-			var next;
+			let maxNotReached = true;
 			let fileList = [];
 
 			do {
 				// load data
 				let awsReturn = await awsListObjects(this, bucket, path, next ? next : null);
 
+				// add to return list
+				fileList = fileList.concat(awsReturn.list);
+
 				// set next token
 				next = awsReturn.next;
 
-				// add to return list
-				fileList = fileList.concat(awsReturn.list);
-			} while (next);
+				// calculate max reached
+				maxNotReached = max && fileList.length < max ? true : false;
+			} while (next && maxNotReached);
 
 			// return list
-			return Promise.resolve(fileList);
+			return Promise.resolve({ list: fileList, next });
 		} else if (uri.substr(0, 5).toLowerCase() == 'gs://') {
 			// google cloud storage
 			structure = uri.substr(5).split('/');