Merge pull request #32 from hotosm/develop

tms endpoint + very basic auth resolves #20

README.md

@@ -26,13 +26,27 @@ Starting the backgound worker:
 - `AWS_SECRET_KEY_ID` - set AWS secret key id
 - `AWS_SECRET_ACCESS_KEY` - set AWS secret access key
 - `DBURI` - set Mongo DB URI
+- `SECRET_TOKEN` - The token used for post requests to `/tms` endpoint
 
 ### Endpoints
 
 -  `/meta` -XGET
 -  `/meta/add` -XPOST
 -  `/providers` -XGET
 -  `/providers/add` -XPOST
+-  `/tms` -XGET
+-  `/tms` -XPOST
+
+### POST parameters for `/tms`:
+To add/update tms endpoint, the following json format should be used:
+
+```json
+{ "uri": "http://example.com/tms_uri",
+  "images": [
+    {"uuid": "http://example.com/image_uri.tif"}
+    ]
+}
+```
 
 ### Search parameters for `/meta`:
 

controllers/analytics.js

@@ -25,6 +25,11 @@ module.exports.query = function (page, limit, cb) {
   });
 };
 
+/**
+* Add a analytics record to the database.
+*
+* @param {int} count - The number of images in the system
+*/
 module.exports.addAnalyticsRecord = function (count) {
   var record = new Analytics({ date: Date.now(), count: count });
   record.save(function (err, record) {
@@ -34,6 +39,25 @@ module.exports.addAnalyticsRecord = function (count) {
   });
 };
 
+/**
+* Check the analytics collection to find the last time the system was updated.
+*
+* @param {dataCallback} cb - A callback with format (error, date)
+*/
+module.exports.getLastUpdateTime = function (cb) {
+  Analytics.findOne().sort({ date: -1 }).exec(function (err, record) {
+    if (err) {
+      return cb(err, null);
+    }
+    // If we don't have a date (this should never happen in practice), set it
+    // to some time in the past
+    if (record === null) {
+      record = { date: new Date('01-01-1970') };
+    }
+    cb(null, record.date);
+  });
+};
+
 /**
  * The date callback that returns the error and date.
  *

controllers/meta.js

@@ -4,6 +4,7 @@ var _ = require('lodash');
 var request = require('request');
 var parse = require('wellknown');
 var bboxPolygon = require('turf-bbox-polygon');
+var Boom = require('boom');
 var Meta = require('../models/meta.js');
 
 /**
@@ -112,17 +113,20 @@ module.exports.query = function (payload, page, limit, cb) {
 * Add Meta Information from a provided URI. This function reads the remote json meta file
 * and adds the content to Meta model.
 * @param {String} remoteUri - a URI to the remote file
+* @param {Date} lastModified
+* @param {Date} lastSystemUpdate
 * @param {Callback} cb - The callback that handles the response
 */
-module.exports.addRemoteMeta = function (remoteUri, cb) {
+module.exports.addRemoteMeta = function (remoteUri, lastModified, lastSystemUpdate, cb) {
   // Check if the meta data is already added
   Meta.findOne({meta_uri: remoteUri}, function (err, meta) {
     if (err) {
       return cb(err);
     }
 
-    // if the meta file doesn't exist then add
-    if (meta === null) {
+    // if the meta file doesn't exist then add, if the meta file is more recent
+    // than our last update, then update
+    if (meta === null || lastModified.getTime() > lastSystemUpdate.getTime()) {
       request(remoteUri, function (err, response, body) {
         if (err) {
           return cb(err);
@@ -135,19 +139,66 @@ module.exports.addRemoteMeta = function (remoteUri, cb) {
           payload.geojson = parse(payload.footprint);
           payload.geojson.bbox = payload.bbox;
 
-          var record = new Meta(payload);
-          record.save(function (err, record) {
+          var query = { uuid: payload.uuid };
+          var options = { upsert: true, new: true, select: { uuid: 1 } };
+          Meta.findOneAndUpdate(query, payload, options, function (err, record) {
             if (err) {
               return cb(err);
             }
-            cb(err, record.uuid + ' added!');
+
+            var status = (meta === null) ? ' added' : ' updated';
+            cb(err, record.uuid + status + '!');
           });
         }
       });
     }
   });
 };
 
+/**
+* Add or Update TMS endpoints related to each image
+* @param {String} remoteUri - a URI to the remote file
+* @param {String} tmsUri - a URI to the TMS
+* @param {Callback} cb - The callback that handles the response
+*/
+module.exports.addUpdateTms = function (remoteUri, tmsUri, cb) {
+  // Check if the meta data is already added
+  Meta.findOne({uuid: remoteUri}, function (err, meta) {
+    if (err) {
+      return cb(err);
+    }
+
+    // if the meta file doesn't exist then add, if the meta file is more recent
+    // than our last update, then update
+    if (meta === null) {
+      err = Boom.create(
+        400,
+        'Image was not found in the catalog!',
+        { timestamp: Date.now() }
+      );
+      return cb(err);
+    } else {
+
+      var custom = [];
+
+      if (typeof meta.custom_tms === 'undefined') {
+        custom.push(tmsUri);
+      } else if (meta.custom_tms.indexOf(tmsUri) === -1) {
+        custom = meta.custom_tms;
+        custom.push(tmsUri);
+      } else {
+        // return if the tms uri is already added to the image
+        return cb(err, meta);
+      }
+
+      // Update the image with the tms uri provided
+      Meta.update({uuid: remoteUri}, {custom_tms: custom}, function (err) {
+        return cb(err, meta);
+      });
+    }
+  });
+};
+
 /**
  * The records callback that returns the error and records.
  *

controllers/tms.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var async = require('async');
+var Model = require('../models/tms.js');
+var meta = require('./meta.js');
+
+/**
+* Query TMS model. Implements all protocols supported by /meta endpoint
+*
+* @param {Object} payload - Payload contains query paramters and their values
+* @param {recordsCallback} cb - The callback that returns the records
+*/
+module.exports.query = function (payload, page, limit, cb) {
+  // bounding box search | looks for bbox in payload
+
+  var skip = limit * (page - 1);
+
+  // Execute the search and return the result via callback
+  Model.count(payload, function (err, count) {
+    if (err) {
+      return cb(err, null, null);
+    }
+    Model.find(payload, null, { skip: skip, limit: limit }).exec(function (err, records) {
+      cb(err, records, count);
+    });
+  });
+};
+
+module.exports.addUpdate = function (payload, cb) {
+
+  var images = [];
+
+  async.each(payload.images, function (image, callback) {
+    meta.addUpdateTms(image.uuid, payload.uri, function (err, meta) {
+      images.push(meta);
+      return callback(err);
+    });
+  }, function (err) {
+    if (err) {
+      return cb(err);
+    }
+
+    var options = { upsert: true, new: true };
+    var query = { uuid: payload.uri };
+    payload.images = images;
+    Model.findOneAndUpdate(query, payload, options, function (err, record) {
+      if (err) {
+        return cb(err);
+      }
+
+      cb(err, record);
+    });
+
+  });
+};

models/meta.js

@@ -3,8 +3,8 @@
 var mongoose = require('mongoose');
 
 var metaSchema = new mongoose.Schema({
-    uuid: {type: String, unique: true, required: true, dropDups: true },
-    meta_uri: {type: String, unique: true, required: false },
+    uuid: {type: String, unique: true, required: true, dropDups: true }, // The URI of the image
+    meta_uri: {type: String, unique: true, required: false }, // To URI of the meta of the image
     thumb_uri: String,
     title: String,
     projection: String,
@@ -20,7 +20,8 @@ var metaSchema = new mongoose.Schema({
     provider: String,
     contact: String,
     geojson: {type: mongoose.Schema.Types.Mixed, index: '2dsphere'},
-    properties: mongoose.Schema.Types.Mixed
+    properties: mongoose.Schema.Types.Mixed,
+    custom_tms: mongoose.Schema.Types.Mixed
 });
 
 module.exports = mongoose.model('meta', metaSchema);

models/tms.js

@@ -0,0 +1,13 @@
+'use strict';
+
+var mongoose = require('mongoose');
+
+var tmsSchema = new mongoose.Schema({
+    uri: {type: String, unique: true, required: true }, // The URI of TMS
+    images: [{
+        uuid: {type: String, required: true}
+    }],
+    created_at: Date
+});
+
+module.exports = mongoose.model('tms', tmsSchema);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "oam-catalog",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "A catalog for Open Aerial Map Imagery",
   "main": "index.js",
   "scripts": {
@@ -17,11 +17,12 @@
   },
   "homepage": "https://github.com/hotosm/oam-catalog",
   "dependencies": {
-    "async": "^1.2.1",
+    "async": "^1.3.0",
     "boom": "^2.7.1",
     "envloader": "0.0.2",
     "fs-extra": "^0.18.2",
     "hapi": "^8.4.0",
+    "hapi-auth-bearer-token": "^3.1.1",
     "hapi-router": "^3.0.1",
     "lodash": "^3.8.0",
     "mongoose": "^4.0.2",

routes/meta.js

@@ -1,18 +1,26 @@
 'use strict';
 
 var _ = require('lodash');
+var parse = require('wellknown');
 var Model = require('../models/meta.js');
 var meta = require('../controllers/meta.js');
 
 module.exports = [
   {
     method: 'POST',
-    path: '/meta/add',
+    path: '/meta',
+    config: { auth: 'simple' },
     handler: function (request, reply) {
       var response = {};
 
       if (!_.isEmpty(request.payload) && _.has(request.payload, 'uuid')) {
-        var record = new Model(request.payload);
+        var payload = request.payload;
+
+        // create a geojson object from footprint and bbox
+        payload.geojson = parse(payload.footprint);
+        payload.geojson.bbox = payload.bbox;
+
+        var record = new Model(payload);
         record.save(function (err, record) {
           if (err) {
             console.log(err);

routes/tms.js

@@ -0,0 +1,51 @@
+'use strict';
+
+var _ = require('lodash');
+var Boom = require('boom');
+var tms = require('../controllers/tms.js');
+
+module.exports = [
+  {
+    method: 'POST',
+    path: '/tms',
+    handler: function (request, reply) {
+      if (!_.isEmpty(request.payload) && _.has(request.payload, 'uri') && _.has(request.payload, 'images')) {
+        tms.addUpdate(request.payload, function (err, record) {
+          if (err) {
+            return reply(Boom.badRequest(err));
+          }
+
+          return reply(record);
+        });
+      } else {
+        var err = Boom.create(
+          400,
+          'There is an Error. Fields missing.',
+          { timestamp: Date.now() }
+        );
+        return reply(Boom.badRequest(err));
+      }
+    },
+    config: { auth: 'simple' }
+  },
+  {
+    method: 'GET',
+    path: '/tms',
+    handler: function (request, reply) {
+      var payload = {};
+
+      if (request.query) {
+        payload = request.query;
+      }
+
+      tms.query(payload, request.page, request.limit, function (err, records, count) {
+        if (err) {
+          return reply(err.message);
+        }
+
+        request.count = count;
+        return reply(records);
+      });
+    }
+  }
+];

services/auth.js

@@ -0,0 +1,14 @@
+'use strict';
+
+// This is a place holder with a basic token authentication
+// More advanced authentiction will be added later
+
+module.exports = function (token, callback) {
+  var secretToken = process.env.SECRET_TOKEN || 'insecuretoken';
+
+  if (token === secretToken) {
+    callback(null, true, {token: token});
+  } else {
+    callback(null, false, { token: token });
+  }
+};

services/s3.js

@@ -37,7 +37,7 @@ var S3 = function (secretId, secretKey, bucket) {
 * @param {responseCallback} cb - The callback that handles the response
 * @param {finishedCallback} finished - The callback that handles when reading is done
 */
-S3.prototype.readBucket = function (cb, done) {
+S3.prototype.readBucket = function (lastSystemUpdate, cb, done) {
   var self = this;
   var images = this.client.listObjects(this.params);
 
@@ -51,8 +51,11 @@ S3.prototype.readBucket = function (cb, done) {
       format = format[format.length - 1];
 
       if (format === 'json') {
+        // Get the last time the metadata file was modified so we can determine
+        // if we need to update it.
+        var lastModified = data.Contents[i].LastModified;
         var url = s3.getPublicUrlHttp(self.params.s3Params.Bucket, data.Contents[i].Key);
-        meta.addRemoteMeta(url, function (err, msg) {
+        meta.addRemoteMeta(url, lastModified, lastSystemUpdate, function (err, msg) {
           if (err) {
             return cb(err);
           }