adding option for logPrefix

swrlab · Oct 28, 2020 · 3b64d01 · 3b64d01
1 parent cbef0e0
commit 3b64d01
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 45 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@swrlab/node-storage-wrapper",
-	"version": "0.1.0",
+	"version": "0.2.0",
 	"description": "Wrapping AWS S3, GCP GCS, file storage",
 	"main": "./src/index.js",
 	"engines": {

diff --git a/src/createUrl.js b/src/createUrl.js
@@ -6,11 +6,13 @@
 
 */
 
-module.exports = async function (uri, ttl) {
+module.exports = async function (uri, ttl, logPrefix) {
 	try {
+		logPrefix = logPrefix ? [logPrefix, '>'] : [];
+
 		if (uri.substr(0, 5).toLowerCase() == 's3://') {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.createUrl.aws >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.createUrl.aws >', uri]));
 
 			return Promise.reject('not implemented');
 		} else if (uri.substr(0, 5).toLowerCase() == 'gs://') {
@@ -20,7 +22,7 @@ module.exports = async function (uri, ttl) {
 			let path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.createUrl.gcp >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.createUrl.gcp >', uri]));
 
 			// set config
 			const config = {
@@ -38,13 +40,13 @@ module.exports = async function (uri, ttl) {
 			uri.substr(0, 8).toLowerCase() == 'https://'
 		) {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.createUrl.https >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.createUrl.https >', uri]));
 
 			// return link
 			return Promise.resolve(uri);
 		} else {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.createUrl.local >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.createUrl.local >', uri]));
 
 			return Promise.reject('not implemented');
 		}

diff --git a/src/delete.js b/src/delete.js
@@ -12,8 +12,9 @@ const deleteLocalFile = (that, filePath) =>
 		});
 	});
 
-module.exports = async function (uri) {
+module.exports = async function (uri, logPrefix) {
 	try {
+		logPrefix = logPrefix ? [logPrefix, '>'] : [];
 		let structure, bucket, path;
 
 		if (uri.substr(0, 5).toLowerCase() == 's3://') {
@@ -23,7 +24,7 @@ module.exports = async function (uri) {
 			path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.delete.s3 >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.delete.s3 >', uri]));
 
 			// delete from aws
 			await this.sdk.s3
@@ -42,7 +43,7 @@ module.exports = async function (uri) {
 			path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.delete.gs >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.delete.gs >', uri]));
 
 			// delete from gcp
 			await this.sdk.gs.bucket(bucket).file(path).delete(path);
@@ -54,13 +55,13 @@ module.exports = async function (uri) {
 			uri.substr(0, 8).toLowerCase() == 'https://'
 		) {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.delete.https (not possible) >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.delete.https (not possible) >', uri]));
 
 			// return ok
 			return Promise.resolve();
 		} else {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.delete.local >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.delete.local >', uri]));
 
 			// delete file
 			await deleteLocalFile(this, uri);

diff --git a/src/list.js b/src/list.js
@@ -4,7 +4,7 @@
 
 */
 
-const awsListObjects = async (that, bucket, path, next) => {
+const awsListObjects = async (that, bucket, path, next, logPrefix) => {
 	try {
 		// load list from aws, pass next token (nullable)
 		let files = await that.sdk.s3
@@ -22,10 +22,13 @@ const awsListObjects = async (that, bucket, path, next) => {
 			next: files.IsTruncated ? files.NextContinuationToken : null,
 		});
 	} catch (err) {
-		that.sdk.log('error', [
-			'storage.list.awsListObjects',
-			JSON.stringify({ bucket, path, next, message: err.message, stack: err.stack }),
-		]);
+		that.sdk.log(
+			'error',
+			logPrefix.concat([
+				'storage.list.awsListObjects',
+				JSON.stringify({ bucket, path, next, message: err.message, stack: err.stack }),
+			])
+		);
 		return Promise.reject(err);
 	}
 };
@@ -38,8 +41,9 @@ const listLocalFiles = (that, uri) =>
 		});
 	});
 
-module.exports = async function (uri, max, next) {
+module.exports = async function (uri, max, next, logPrefix) {
 	try {
+		logPrefix = logPrefix ? [logPrefix, '>'] : [];
 		let structure, bucket, path, file;
 
 		if (uri.substr(0, 5).toLowerCase() == 's3://') {
@@ -49,15 +53,15 @@ module.exports = async function (uri, max, next) {
 			path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.list.aws >', uri, max]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.list.aws >', uri, max]));
 
 			// load file
 			let maxNotReached = true;
 			let fileList = [];
 
 			do {
 				// load data
-				let awsReturn = await awsListObjects(this, bucket, path, next ? next : null);
+				let awsReturn = await awsListObjects(this, bucket, path, next ? next : null, logPrefix);
 
 				// add to return list
 				fileList = fileList.concat(awsReturn.list);
@@ -78,7 +82,7 @@ module.exports = async function (uri, max, next) {
 			path = structure.join('/');
 
 			// log request
-			this.sdk.log(this, 'log', ['storage.list.gcp >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.list.gcp >', uri]));
 
 			// load file
 			file = await this.sdk.gs.bucket(bucket).getFiles({
@@ -89,7 +93,7 @@ module.exports = async function (uri, max, next) {
 			return Promise.resolve(file[0]);
 		} else {
 			// log request
-			this.sdk.log(this, 'log', ['storage.list.local >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.list.local >', uri]));
 
 			// local file
 			let file = await listLocalFiles(this, uri);

diff --git a/src/load.js b/src/load.js
@@ -25,8 +25,9 @@ const loadLocalFile = (that, uri) =>
 		});
 	});
 
-module.exports = async function (uri) {
+module.exports = async function (uri, logPrefix) {
 	try {
+		logPrefix = logPrefix ? [logPrefix, '>'] : [];
 		let structure, bucket, path, file;
 
 		if (uri.substr(0, 5).toLowerCase() == 's3://') {
@@ -36,7 +37,7 @@ module.exports = async function (uri) {
 			path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.load.aws >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.load.aws >', uri]));
 
 			// load file
 			let file = await this.sdk.s3
@@ -54,7 +55,7 @@ module.exports = async function (uri) {
 			bucket = structure.shift();
 			path = structure.join('/');
 
-			this.sdk.log(this, 'log', ['storage.load gcp >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.load gcp >', uri]));
 
 			// load file
 			file = await this.sdk.gs.bucket(bucket).file(path).download();
@@ -66,7 +67,7 @@ module.exports = async function (uri) {
 			uri.substr(0, 8).toLowerCase() == 'https://'
 		) {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.load.https >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.load.https >', uri]));
 
 			// public http(s) endpoint
 			let file = await fetch(uri, {
@@ -94,7 +95,7 @@ module.exports = async function (uri) {
 			}
 		} else {
 			// log progress
-			this.sdk.log(this, 'log', ['storage.load.local >', uri]);
+			this.sdk.log(this, 'log', logPrefix.concat(['storage.load.local >', uri]));
 
 			// local file
 			let file = await loadLocalFile(this, uri);

diff --git a/src/move.js b/src/move.js
@@ -4,8 +4,9 @@
 
 */
 
-module.exports = async function (sourceUri, destinationUri, keepOriginal) {
+module.exports = async function (sourceUri, destinationUri, keepOriginal, logPrefix) {
 	try {
+		logPrefix = logPrefix ? [logPrefix, '>'] : [];
 		let structure, bucket, path, blob;
 
 		if (
@@ -22,17 +23,25 @@ module.exports = async function (sourceUri, destinationUri, keepOriginal) {
 			// move file within gcs
 			if (keepOriginal != true) {
 				// move is productive/ destructive
-				this.sdk.log(this, 'log', ['storage.move.gcp2gcp >', sourceUri, destinationUri]);
+				this.sdk.log(
+					this,
+					'log',
+					logPrefix.concat(['storage.move.gcp2gcp >', sourceUri, destinationUri])
+				);
 
 				// move file
 				await this.sdk.gs.bucket(bucket).file(path).move(destinationUri);
 			} else {
 				// mode is dev, only copy file
-				this.sdk.log(this, 'log', [
-					'storage.move.gcp2gcp (only copying) >',
-					sourceUri,
-					destinationUri,
-				]);
+				this.sdk.log(
+					this,
+					'log',
+					logPrefix.concat([
+						'storage.move.gcp2gcp (only copying) >',
+						sourceUri,
+						destinationUri,
+					])
+				);
 
 				// copy file
 				await this.sdk.gs.bucket(bucket).file(path).copy(destinationUri);
@@ -52,7 +61,11 @@ module.exports = async function (sourceUri, destinationUri, keepOriginal) {
 			path = structure.join('/');
 
 			// always copying
-			this.sdk.log(this, 'log', ['storage.move.aws2aws copying >', sourceUri, destinationUri]);
+			this.sdk.log(
+				this,
+				'log',
+				logPrefix.concat(['storage.move.aws2aws copying >', sourceUri, destinationUri])
+			);
 
 			// copy file
 			await this.sdk.s3
@@ -66,7 +79,11 @@ module.exports = async function (sourceUri, destinationUri, keepOriginal) {
 			// move file within gcs
 			if (keepOriginal != true) {
 				// move is productive/ destructive
-				this.sdk.log(this, 'log', ['storage.move.aws2aws deleting source >', sourceUri]);
+				this.sdk.log(
+					this,
+					'log',
+					logPrefix.concat(['storage.move.aws2aws deleting source >', sourceUri])
+				);
 
 				// parse source
 				structure = sourceUri.substr(5).split('/');
@@ -86,7 +103,11 @@ module.exports = async function (sourceUri, destinationUri, keepOriginal) {
 			return Promise.resolve();
 		} else {
 			// any to any transfer
-			this.sdk.log(this, 'log', ['storage.move.any2any >', keepOriginal, sourceUri, destinationUri]);
+			this.sdk.log(
+				this,
+				'log',
+				logPrefix.concat(['storage.move.any2any >', keepOriginal, sourceUri, destinationUri])
+			);
 
 			// download file
 			blob = await this.load(sourceUri);
@@ -98,11 +119,15 @@ module.exports = async function (sourceUri, destinationUri, keepOriginal) {
 			if (keepOriginal != true) {
 				await this.delete(sourceUri);
 			} else {
-				this.sdk.log(this, 'log', [
-					'storage.move.any2any not deleting sourceUri >',
-					keepOriginal,
-					sourceUri,
-				]);
+				this.sdk.log(
+					this,
+					'log',
+					logPrefix.concat([
+						'storage.move.any2any not deleting sourceUri >',
+						keepOriginal,
+						sourceUri,
+					])
+				);
 			}
 
 			// return ok

diff --git a/src/save.js b/src/save.js
@@ -25,8 +25,9 @@ const deleteLocalFile = (that, filePath) =>
 		});
 	});
 
-module.exports = async function (uri, buffer) {
+module.exports = async function (uri, buffer, logPrefix) {
 	try {
+		logPrefix = logPrefix ? [logPrefix, '>'] : [];
 		let structure, bucket, path;
 
 		if (uri.substr(0, 5).toLowerCase() == 's3://') {
@@ -36,7 +37,7 @@ module.exports = async function (uri, buffer) {
 			path = structure.join('/');
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.save.s3 >', uri]);
+			this.sdk.log(this, 'log', [logPrefix ? logPrefix : undefined, 'storage.save.s3 >', uri]);
 
 			// upload to aws
 			await this.sdk.s3
@@ -60,7 +61,7 @@ module.exports = async function (uri, buffer) {
 			await saveLocalFile(this, tempFilePath, buffer);
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.save.gs >', uri]);
+			this.sdk.log(this, 'log', [logPrefix ? logPrefix : undefined, 'storage.save.gs >', uri]);
 
 			// upload file to gcs
 			await this.sdk.gs.bucket(bucket).upload(tempFilePath, {
@@ -78,7 +79,7 @@ module.exports = async function (uri, buffer) {
 			// local file
 
 			// log progress
-			this.sdk.log(this, 'log', ['storage.save.local >', uri]);
+			this.sdk.log(this, 'log', [logPrefix ? logPrefix : undefined, 'storage.save.local >', uri]);
 
 			// save file
 			let file = await saveLocalFile(this, uri, buffer);