Sync updated/deleted meta to their S3 counterparts

.env.sample

@@ -27,6 +27,8 @@ OIN_REGISTER_URL=http://localhost:8080/fixtures/oin-buckets.json
 OIN_BUCKET=oin-hotosm-staging
 OIN_BUCKET_PREFIX=development
 UPLOAD_BUCKET=oam-uploader-staging-temp
+S3_PUBLIC_DOMAIN=s3.amazonaws.com
+
 # Run quickly like this only for non-production
 CRON_TIME=*/15 * * * * *
 GDRIVE_KEY=abc123

.nvmrc

@@ -1 +1 @@
-4.5.0
+6.10.2

catalog-worker.js

@@ -9,12 +9,13 @@ console.log('Starting catalog worker...');
 require('dotenv').config();
 
 var _ = require('lodash');
-var S3 = require('./services/s3.js');
+var S3 = require('aws-sdk/clients/s3');
 var async = require('async');
 var config = require('./config');
-var Conn = require('./services/db.js');
-var analytics = require('./controllers/analytics.js');
-var Meta = require('./models/meta.js');
+var Conn = require('./services/db');
+var analytics = require('./controllers/analytics');
+var meta = require('./controllers/meta');
+var Meta = require('./models/meta');
 // Replace mongoose's deprecated promise library (mpromise) with bluebird
 var mongoose = require('mongoose');
 mongoose.Promise = require('bluebird');
@@ -105,9 +106,38 @@ var readBuckets = function (tasks) {
     });
 };
 
-/**
- * The main function to get the registered buckets, read them and update metadata
- */
+// Read bucket method for S3. It reads the S3 bucket and adds/updates *_metadata.json to Meta model
+var readBucket = function (bucket, lastSystemUpdate, errCb, done) {
+  console.info('--- Reading from bucket: ' + bucket.bucket_name + ' ---');
+
+  var s3 = new S3();
+  s3.listObjects({
+    Bucket: config.oinBucket,
+    Prefix: config.oinBucketPrefix
+  }, function (err, data) {
+    if (err) {
+      errCb(err);
+      done(err);
+      return;
+    }
+    var tasks = [];
+    data.Contents.forEach(function (item) {
+      if (item.Key.includes('_meta.json')) {
+        // Get the last time the metadata file was modified so we can determine
+        // if we need to update it.
+        var lastModified = item.LastModified;
+        var url = `https://${config.s3PublicDomain}/${bucket.bucket_name}/${item.Key}`;
+        var task = function (done) {
+          meta.addRemoteMeta(url, lastModified, lastSystemUpdate, done);
+        };
+        tasks.push(task);
+      }
+    });
+    done(null, tasks);
+  });
+};
+
+// The main function to get the registered buckets, read them and update metadata
 var getListAndReadBuckets = function () {
   // Start off by getting the last time the system was updated.
   analytics.getLastUpdateTime(function (err, lastSystemUpdate) {
@@ -125,8 +155,7 @@ var getListAndReadBuckets = function () {
       var tasks = _.map(buckets, function (bucket) {
         return function (done) {
           if (bucket.type === 's3') {
-            var s3 = new S3(bucket.bucket_name);
-            s3.readBucket(lastSystemUpdate, consoleLog, done);
+            readBucket(bucket, lastSystemUpdate, consoleLog, done);
           } else {
             console.error('Unknown bucket type: ' + bucket.type);
           }

config.js

@@ -51,6 +51,9 @@ const config = {
   oinBucketPrefix: process.env.OIN_BUCKET_PREFIX,
   // OIN bucket for temporary storage of direct uploads
   uploadBucket: process.env.UPLOAD_BUCKET,
+  // Base domain for public read access to OIN bucket
+  s3PublicDomain: process.env.S3_PUBLIC_DOMAIN,
+
   // How often to poll OIN buckets for new imagery
   cronTime: process.env.CRON_TIME,
   // Location of master record of OIN buckets to poll

models/meta.js

@@ -3,6 +3,8 @@
 var mongoose = require('mongoose');
 var Joi = require('joi');
 
+var S3Sync = require('../services/s3_sync');
+
 var metaSchema = new mongoose.Schema({
   // The URI of the image
   uuid: {type: String, unique: true, required: true, dropDups: true},
@@ -60,4 +62,32 @@ metaSchema.statics = {
   }
 };
 
+metaSchema.methods = {
+
+  // Update a metadata object only after the updates have been synced to the corelating
+  // _meta.json file on S3.
+  oamUpdate: function (newParams, callback) {
+    var s3Sync = new S3Sync(this.meta_uri);
+    s3Sync.updateRemoteMeta(newParams, () => {
+      let updatedMeta = Object.assign(this, newParams);
+      updatedMeta.save(function (err) {
+        if (err) throw new Error('Error saving meta: ', err);
+        callback();
+      });
+    });
+  },
+
+  // Delete a metadata object only after its corelating _meta.json file has
+  // been deleted on S3.
+  oamDelete: function (callback) {
+    var s3Sync = new S3Sync(this.meta_uri);
+    s3Sync.deleteRemoteMeta(() => {
+      this.remove(function (err) {
+        if (err) throw new Error('Error deleting meta: ', err);
+        callback();
+      });
+    });
+  }
+};
+
 module.exports = mongoose.model('Meta', metaSchema);

routes/meta.js

@@ -49,7 +49,7 @@ module.exports = [
 
       metaController.query(payload, request.page, request.limit, function (err, records, count) {
         if (err) {
-          console.log(err);
+          console.error(err);
           return reply(err.message);
         }
 
@@ -78,6 +78,7 @@ module.exports = [
 
       Meta.findOne({_id: metaId}, function (err, record) {
         if (err) {
+          console.error(err);
           return reply(Boom.badImplementation(err.message));
         }
         return reply(record);
@@ -105,9 +106,10 @@ module.exports = [
       ]
     },
     handler: function (request, reply) {
-      const metaId = request.app.requestedObject._id;
-      Meta.update({_id: metaId}, request.payload, function (err, _result) {
+      let meta = request.app.requestedObject;
+      meta.oamUpdate(request.payload, function (err, _result) {
         if (err) {
+          console.error(err);
           reply(Boom.badImplementation(err));
           return;
         }
@@ -136,9 +138,10 @@ module.exports = [
       ]
     },
     handler: function (request, reply) {
-      const metaId = request.app.requestedObject._id;
-      Meta.findByIdAndRemove(metaId, function (err, _result) {
+      let meta = request.app.requestedObject;
+      meta.oamDelete(function (err, _result) {
         if (err) {
+          console.error(err);
           reply(Boom.badImplementation(err));
           return;
         }

routes/uploads.js

@@ -5,19 +5,13 @@ var ObjectID = require('mongodb').ObjectID;
 var queue = require('queue-async');
 var Boom = require('boom');
 var Joi = require('joi');
-var AWS = require('aws-sdk');
+var S3 = require('aws-sdk/clients/s3');
 
 var Meta = require('../models/meta');
 var config = require('../config');
 
 var sendgrid = require('sendgrid')(config.sendgridApiKey);
 
-AWS.config = {
-  region: config.awsRegion,
-  accessKeyId: config.awsKey,
-  secretAccessKey: config.awsSecret
-};
-
 var uploadSchema = Meta.getSceneValidations();
 
 function insertImages (db, scene, userID, callback) {
@@ -100,7 +94,7 @@ module.exports = [
     },
     handler: function (request, reply) {
       var payload = JSON.parse(request.payload);
-      var s3 = new AWS.S3();
+      var s3 = new S3();
       var params = {
         Bucket: config.uploadBucket,
         Key: payload.name,