Implement transcoding with monq (queue)

Procfile

@@ -1,2 +1,3 @@
 web: node index.js
 worker: node catalog-worker.js
+transcoder: node bin/transcoder.js

bin/transcoder.js

@@ -0,0 +1,25 @@
+var monq = require('monq');
+
+var config = require('../config');
+var transcoder = require('../services/transcoder');
+
+var client = monq(config.dbUri);
+
+var worker = client.worker(['transcoder']);
+
+worker.register({
+  transcode: (params, callback) =>
+    transcoder.transcode(
+      params.sourceUrl,
+      params.output,
+      params.metaUrl,
+      callback
+    )
+});
+
+worker.on('dequeued', data => console.log('dequeued:', data));
+worker.on('failed', data => console.error('failed:', data));
+worker.on('complete', data => console.log('complete:', data));
+worker.on('error', data => console.error('error:', data));
+
+worker.start();

config.js

@@ -9,15 +9,13 @@ if (process.env.NODE_ENV === 'test') {
   process.env.NEW_RELIC_ENABLED = false;
   process.env.PORT = 47357;
   process.env.API_ENDPOINT = 'http://localhost:' + process.env.PORT;
-  process.env.IMAGE_PROCESSOR = '../test/specs/_process-image_stub';
 }
 
 // Amendments for integration tests, the ones that run in Docker against
 // a S3 bucket.
 if (process.env.INTEGRATION_TESTS === 'true') {
   process.env.PORT = 4000;
   process.env.API_ENDPOINT = 'http://localhost:' + process.env.PORT;
-  process.env.IMAGE_PROCESSOR = './process-image';
 }
 
 if (process.env.NODE_ENV === 'production') {
@@ -39,11 +37,6 @@ const config = {
   // DB connection
   dbUri: process.env.DB_URI,
 
-  // Not ideal, but Hapi's plugin architecture makes it hard to stub modules
-  // the normal tools like sinon or proxyquire. So here we manually specify
-  // the path to the image processor code.
-  imageProcessorPath: process.env.IMAGE_PROCESSOR || './process-image',
-
   // OIN bucket in which imagery ultimately lives
   oinBucket: process.env.OIN_BUCKET,
   // Place imagery in a folder. Useful for running multiple OAM instances

controllers/meta.js

@@ -134,7 +134,7 @@ module.exports.addRemoteMeta = function (remoteUri, lastModified, lastSystemUpda
 
     // if the meta file doesn't exist then add, if the meta file is more recent
     // than our last update, then update
-    if (meta === null || lastModified > lastSystemUpdate) {
+    if (meta == null || lastModified > lastSystemUpdate) {
       return request({
         json: true,
         uri: remoteUri
@@ -143,6 +143,10 @@ module.exports.addRemoteMeta = function (remoteUri, lastModified, lastSystemUpda
           return cb(err);
         }
 
+        // clean up metadata that conflicts with Mongo in the staging bucket
+        delete payload.__v;
+        delete payload._id;
+
         if (response.statusCode === 200 && payload != null) {
           if (payload.uuid == null) {
             // not OIN metadata
@@ -154,8 +158,10 @@ module.exports.addRemoteMeta = function (remoteUri, lastModified, lastSystemUpda
 
           // create a geojson object from footprint and bbox
           // TODO: Put in a Mongoose middleware hook
-          payload.geojson = parse(payload.footprint);
-          payload.geojson.bbox = payload.bbox;
+          if (payload.footprint != null && payload.geojson == null) {
+            payload.geojson = parse(payload.footprint);
+            payload.geojson.bbox = payload.bbox;
+          }
 
           var query = { uuid: payload.uuid };
           var options = { upsert: true, new: true, select: { uuid: 1 } };

docker-compose.yml

@@ -55,6 +55,32 @@ services:
       - ./index.js:/app/index.js
       - ./newrelic.js:/app/newrelic.js
       - ./package.json:/app/package.json
+  transcoder:
+    environment:
+      - DB_URI=mongodb://mongo/oam-api
+    env_file: .env
+    build:
+      context: .
+      args:
+        - NODE_ENV=development
+    command: nodemon bin/transcoder.js
+    depends_on:
+      - mongo
+    links:
+      - mongo
+    volumes:
+      - ./bin:/app/bin
+      - ./controllers:/app/controllers
+      - ./models:/app/models
+      - ./plugins:/app/plugins
+      - ./routes:/app/routes
+      - ./services:/app/services
+      - ./test:/app/test
+      - ./catalog-worker.js:/app/catalog-worker.js
+      - ./config.js:/app/config.js
+      - ./index.js:/app/index.js
+      - ./newrelic.js:/app/newrelic.js
+      - ./package.json:/app/package.json
   register:
     build:
       context: .

models/meta.js

@@ -64,6 +64,20 @@ metaSchema.statics = {
 
 metaSchema.methods = {
 
+  oamSync: function (callback) {
+    var s3Sync = new S3Sync(this.meta_uri);
+    var meta = Object.assign({}, this._doc);
+
+    // remove MongoDB attributes
+    delete meta.__v;
+    delete meta._id;
+
+    // remove internal tracking
+    delete meta.meta_uri;
+
+    s3Sync.uploadMeta(JSON.stringify(meta)).then(callback).catch(callback);
+  },
+
   // Update a metadata object only after the updates have been synced to the corelating
   // _meta.json file on S3.
   oamUpdate: function (newParams, callback) {