Add _meta.json URI during imagery conversion

.travis.yml

@@ -9,6 +9,7 @@ services:
 env:
   global:
     # Set OAM_DEBUG=true in Travis UI to see debug output
+    - OAM_DEBUG=true
     - NODE_ENV=test
     - GH_REF=github.com/hotosm/oam-catalog.git
     - PRODUCTION_BRANCH=master

services/s3_sync.js

@@ -21,8 +21,9 @@ module.exports = class S3Sync {
 
   // Don't try to manipulate metadata files that aren't in OIN's public bucket.
   isOINMeta () {
-    // TODO: Also match the S3 domain for more rigorous matching.
-    return this.remoteMetaUri.includes(`/${config.oinBucket}/`);
+    let isInDomain = this.remoteMetaUri.includes(`${config.oinBucket}.${config.s3PublicDomain}`);
+    let isInPath = this.remoteMetaUri.includes(`/${config.oinBucket}/`);
+    return isInDomain || isInPath;
   }
 
   updateRemoteMeta (newDetails, callback) {

test/docker-compose.yml

@@ -23,6 +23,7 @@ services:
       # locally, the codebase is mounted from the host, including
       # the `.env` file, so the `dotenv` npm module takes responsibility
       # for loading these values.
+      - OAM_DEBUG
       - AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY
       - FACEBOOK_APP_ID

test/helper.js

@@ -106,6 +106,17 @@ module.exports = {
   // This comes from the periodic cron to check the bucket for
   // new imagery. Once a new image's *meta.json is found it is
   // parsed and added to the DB.
+  // TODO:
+  //   This is no longer necessary as the _meta.json URI is actually
+  //   added at the point of conversion. Before the conversion step
+  //   added the metadata URI we had to wait for the background cron
+  //   worker process to index new metadata files from S3. This was
+  //   good for testing, because it meant that this function was
+  //   testing that background worker process (`/catalog-worker.js`
+  //   at time of writing). I'm not aware that this indexing step
+  //   adds anything at all new that we can detect. Therefore the
+  //   best solution would just be to write dedicated tests for the
+  //   indexing process.
   waitForIndexing: function (title, processedCb) {
     var getOptions = {
       url: config.apiEndpoint + '/meta?title=' + title,

worker/process-image.js

@@ -90,14 +90,17 @@ function _processImage (scene, sourceUrl, targetPrefix, callback) {
 
     log(['debug'], 'Converted image to OAM standard format. Input: ', sourceUrl, 'Output: ', output);
 
+    var metaUri = `http://${s3bucket}.s3.amazonaws.com/${targetPrefix}_meta.json`;
+
     return request.get({
       json: true,
-      uri: `http://${s3bucket}.s3.amazonaws.com/${targetPrefix}_meta.json`
+      uri: metaUri
     }, function (err, rsp, metadata) {
       if (err) {
         return _callback(err);
       }
 
+      metadata.meta_uri = metaUri;
       return _callback(null, {
         metadata: metadata
       });