#505 Dynamic Vector Layers (#506)

* #505 Dynamic Vector Layer querying 1

* #505 Time queries, reselect improvement, download extent or full

* #505 Dynamic vector layers

API/Backend/Geodatasets/routes/geodatasets.js

@@ -13,8 +13,9 @@ const Geodatasets = geodatasets.Geodatasets;
 const makeNewGeodatasetTable = geodatasets.makeNewGeodatasetTable;
 
 //Returns a geodataset table as a geojson
-router.post("/get", function (req, res, next) {
-  get("post", req, res, next);
+router.get("/get/:layer", function (req, res, next) {
+  req.query.layer = req.params.layer;
+  get("get", req, res, next);
 });
 router.get("/get", function (req, res, next) {
   get("get", req, res, next);
@@ -51,10 +52,66 @@ function get(reqtype, req, res, next) {
       if (result) {
         let table = result.dataValues.table;
         if (type == "geojson") {
+          let q = `SELECT properties, ST_AsGeoJSON(geom) FROM ${table}`;
+
+          let hasBounds = false;
+          let minx = req.query?.minx;
+          let miny = req.query?.miny;
+          let maxx = req.query?.maxx;
+          let maxy = req.query?.maxy;
+          if (minx != null && miny != null && maxx != null && maxy != null) {
+            // ST_MakeEnvelope is (xmin, ymin, xmax, ymax, srid)
+            q += ` WHERE ST_Intersects(ST_MakeEnvelope(${parseFloat(
+              minx
+            )}, ${parseFloat(miny)}, ${parseFloat(maxx)}, ${parseFloat(
+              maxy
+            )}, 4326), geom)`;
+            hasBounds = true;
+          }
+          if (req.query?.endtime != null) {
+            const format = req.query?.format || "YYYY-MM-DDTHH:MI:SSZ";
+            let t = ` `;
+            if (!hasBounds) t += `WHERE `;
+            else t += `AND `;
+
+            if (
+              req.query?.startProp == null ||
+              req.query?.startProp.indexOf(`'`) != -1 ||
+              req.query?.endProp == null ||
+              req.query?.endProp.indexOf(`'`) != -1 ||
+              req.query?.starttime == null ||
+              req.query?.starttime.indexOf(`'`) != -1 ||
+              req.query?.endtime == null ||
+              req.query?.endtime.indexOf(`'`) != -1 ||
+              format.indexOf(`'`) != -1
+            ) {
+              res.send({
+                status: "failure",
+                message: "Missing inner or malformed 'time' parameters.",
+              });
+              return;
+            }
+
+            // prettier-ignore
+            t += [
+              `(`,
+                `properties->>'${req.query.startProp}' IS NOT NULL AND properties->>'${req.query.endProp}' IS NOT NULL AND`, 
+                  ` date_part('epoch', to_timestamp(properties->>'${req.query.startProp}', '${format}'::text)) >= date_part('epoch', to_timestamp('${req.query.starttime}'::text, '${format}'::text))`,
+                  ` AND date_part('epoch', to_timestamp(properties->>'${req.query.endProp}', '${format}'::text)) <= date_part('epoch', to_timestamp('${req.query.endtime}'::text, '${format}'::text))`,
+              `)`,
+              ` OR `,
+              `(`,
+                `properties->>'${req.query.startProp}' IS NULL AND properties->>'${req.query.endProp}' IS NOT NULL AND`,
+                  ` date_part('epoch', to_timestamp(properties->>'${req.query.endProp}', '${format}'::text)) >= date_part('epoch', to_timestamp('${req.query.starttime}'::text, '${format}'::text))`,
+                  ` AND date_part('epoch', to_timestamp(properties->>'${req.query.endProp}', '${format}'::text)) >=  date_part('epoch', to_timestamp('${req.query.endtime}'::text, '${format}'::text))`,
+              `)`
+            ].join('')
+            q += t;
+          }
+          q += `;`;
+
           sequelize
-            .query(
-              "SELECT properties, ST_AsGeoJSON(geom)" + " " + "FROM " + table
-            )
+            .query(q)
             .then(([results]) => {
               let geojson = { type: "FeatureCollection", features: [] };
               for (let i = 0; i < results.length; i++) {
@@ -299,10 +356,35 @@ router.post("/search", function (req, res, next) {
     });
 });
 
+router.post("/append/:name", function (req, res, next) {
+  req.body = {
+    name: req.params.name,
+    geojson: req.body,
+    action: "append",
+  };
+  recreate(req, res, next);
+});
+
+router.post("/recreate/:name", function (req, res, next) {
+  req.body = {
+    name: req.params.name,
+    geojson: req.body,
+    action: "recreate",
+  };
+  recreate(req, res, next);
+});
+
 router.post("/recreate", function (req, res, next) {
+  recreate(req, res, next);
+});
+
+function recreate(req, res, next) {
   let features = null;
   try {
-    features = JSON.parse(req.body.geojson).features;
+    features =
+      typeof req.body.geojson === "string"
+        ? JSON.parse(req.body.geojson).features
+        : req.body.geojson.features;
   } catch (err) {
     logger("error", "Failure: Malformed file.", req.originalUrl, req, err);
     res.send({
@@ -330,7 +412,7 @@ router.post("/recreate", function (req, res, next) {
       }
 
       let drop_qry = "TRUNCATE TABLE " + result.table + " RESTART IDENTITY";
-      if (req.body.hasOwnProperty("action") && req.body.action=="append") {
+      if (req.body.hasOwnProperty("action") && req.body.action == "append") {
         drop_qry = "";
       }
 
@@ -360,40 +442,40 @@ router.post("/recreate", function (req, res, next) {
       res.send(result);
     }
   );
+}
 
-  function populateGeodatasetTable(Table, features, cb) {
-    let rows = [];
-
-    for (var i = 0; i < features.length; i++) {
-      rows.push({
-        properties: features[i].properties,
-        geometry_type: features[i].geometry.type,
-        geom: {
-          crs: { type: "name", properties: { name: "EPSG:4326" } },
-          type: features[i].geometry.type,
-          coordinates: features[i].geometry.coordinates,
-        },
-      });
-    }
+function populateGeodatasetTable(Table, features, cb) {
+  let rows = [];
 
-    Table.bulkCreate(rows, { returning: true })
-      .then(function (response) {
-        cb(true);
-        return null;
-      })
-      .catch(function (err) {
-        logger(
-          "error",
-          "Geodatasets: Failed to populate a geodataset table!",
-          req.originalUrl,
-          req,
-          err
-        );
-        cb(false);
-        return null;
-      });
+  for (var i = 0; i < features.length; i++) {
+    rows.push({
+      properties: features[i].properties,
+      geometry_type: features[i].geometry.type,
+      geom: {
+        crs: { type: "name", properties: { name: "EPSG:4326" } },
+        type: features[i].geometry.type,
+        coordinates: features[i].geometry.coordinates,
+      },
+    });
   }
-});
+
+  Table.bulkCreate(rows, { returning: true })
+    .then(function (response) {
+      cb(true);
+      return null;
+    })
+    .catch(function (err) {
+      logger(
+        "error",
+        "Geodatasets: Failed to populate a geodataset table!",
+        req.originalUrl,
+        req,
+        err
+      );
+      cb(false);
+      return null;
+    });
+}
 
 function tile2Lng(x, z) {
   return (x / Math.pow(2, z)) * 360 - 180;

config/js/config.js

@@ -2796,6 +2796,9 @@ function layerPopulateVariable(modalId, layerType) {
         value: "Example",
       },
     ];
+    currentLayerVars.dynamicExtent = currentLayerVars.dynamicExtent || false;
+    currentLayerVars.dynamicExtentMoveThreshold =
+      currentLayerVars.dynamicExtentMoveThreshold || null;
     currentLayerVars.shortcutSuffix = currentLayerVars.shortcutSuffix || null;
 
     if (layerType == "tile") {

config/js/keys.js

@@ -42,6 +42,18 @@ var Keys = {
                 "<div class='example_title'>Uploading CSVs</div>",
                 '<div>curl -i -X POST -H "Authorization:Bearer <span>&lt;token&gt;</span>" -F "name={dataset_name}" -F "upsert=true" -F "header=[\"File\",\"Target\",\"ShotNumber\",\"Distance(m)\",\"LaserPower\",\"SpectrumTotal\",\"SiO2\",\"TiO2\",\"Al2O3\",\"FeOT\",\"MgO\",\"CaO\",\"Na2O\",\"K2O\",\"Total\",\"SiO2_RMSEP\",\"TiO2_RMSEP\",\"Al2O3_RMSEP\",\"FeOT_RMSEP\",\"MgO_RMSEP\",\"CaO_RMSEP\",\"Na2O_RMSEP\",\"K2O_RMSEP\"]" -F "data=@{path/to.csv};type=text/csv" ' + location.origin + '/api/datasets/upload</div>',
               "</li>",
+              "<li class='row'>",
+                "<div class='example_title'>Upserting a Geodataset</div>",
+                '<div>curl -i -X POST -H "Authorization:Bearer <span>&lt;token&gt;</span>" -H "Content-Type: application/json" --data-binary "@{path_to_your_file}.json" ' + location.origin + '/api/geodatasets/recreate/{geodataset_name}</div>',
+              "</li>",
+              "<li class='row'>",
+                "<div class='example_title'>Appending to an existing Geodataset</div>",
+                '<div>curl -i -X POST -H "Authorization:Bearer <span>&lt;token&gt;</span>" -H "Content-Type: application/json" --data-binary "@{path_to_your_file_to_append}.json" ' + location.origin + '/api/geodatasets/append/{geodataset_name}</div>',
+              "</li>",
+              "<li class='row'>",
+                "<div class='example_title'>Get an existing Geodataset</div>",
+                '<div>curl -i -X GET -H "Authorization:Bearer <span>&lt;token&gt;</span>" -H "Content-Type: application/json" ' + location.origin + '/api/geodatasets/get/{geodataset_name}</div>',
+              "</li>",
             "</ul>",
           "</div>",
         "</div>"

docs/pages/Configure/Layers/Vector/Vector.md

@@ -27,7 +27,7 @@ A file path that points to a geojson. If the path is relative, it will be relati
 
 #### Controlled
 
-_type:_ bool
+_type:_ bool  
 Whether the layer can be dynamically updated or not. If true, the layer can be dynamically updated and the URL is not required.
 
 If true and a URL is set and Time Enabled is true, the initial url query will be performed.
@@ -127,6 +127,8 @@ Example:
 ```javascript
 {
     "useKeyAsName": "propKey || [propKey1, propKey2, ...]",
+    "dynamicExtent": false,
+    "dynamicExtentMoveThreshold": "100000000/z",
     "shortcutSuffix": "single letter to 'ATL + {letter}' toggle the layer on and off",
     "hideMainFeature": false,
     "datasetLinks": [
@@ -254,6 +256,8 @@ Example:
 ```
 
 - `useKeyAsName`: The property key whose value should be the hover text of each feature. If left unset, the hover key and value will be the first one listed in the feature's properties. This may also be an array of keys.
+- `dynamicExtent`: If true, tries to only query the vector features present in the user's current map viewport. Pan and zooming causes requeries. If used with a geodataset, the time and extent queries will work out-of-the-box. Otherwise, if using an external server, the following parameters in `{}` will be automatically replaced on query in the url: `starttime={starttime}&endtime={endtime}&startprop={startprop}&endprop={endprop}&crscode={crscode}&zoom={zoom}&minx={minx}&miny={miny}&maxx={maxx}&maxy={maxy}`
+- `dynamicExtentMoveThreshold`: If `dynamicExtent` is true, only requery if the map was panned past the stated threshold. Unit is in meters. If a zoom-dependent threshold is desired, set this value to a string ending in `/z`. This will then internally use `dynamicExtentMoveThreshold / Math.pow(2, zoom)` as the threshold value.
 - `shortcutSuffix`: A single letter to 'ALT + {letter}' toggle the layer on and off. Please verify that your chosen shortcut does not conflict with other system or browser-level keyboard shortcuts.
 - `hideMainFeature`: If true, hides all typically rendered features. This is useful if showing only `*Attachments` sublayers is desired. Default false
 - `datasetLinks`: Datasets are csvs uploaded from the "Manage Datasets" page accessible on the lower left. Every time a feature from this layer is clicked with datasetLinks configured, it will request the data from the server and include it with it's regular geojson properties. This is especially useful when single features need a lot of metadata to perform a task as it loads it only as needed.

src/essence/Ancillary/Description.js

@@ -70,6 +70,11 @@ const Description = {
 
         this.inited = true
         if (this.waitingOnUpdate) this.updateInfo()
+
+        $(window).on('resize', () => {
+            $('#mainDescPointLinks > dl.dropy').removeClass('open')
+            $(`#mainDescPointLinks_global`).empty()
+        })
     },
     updateInfo(force) {
         if (force !== true) {

src/essence/Ancillary/TimeControl.js

@@ -292,7 +292,11 @@ var TimeControl = {
         var reloadedLayers = []
         for (let layerName in L_.layers.data) {
             const layer = L_.layers.data[layerName]
-            if (layer.time && layer.time.enabled === true) {
+            if (
+                layer.time &&
+                layer.time.enabled === true &&
+                layer.variables?.dynamicExtent != true
+            ) {
                 TimeControl.reloadLayer(layer)
                 reloadedLayers.push(layer.name)
             }

src/essence/Ancillary/TimeUI.js

@@ -400,6 +400,7 @@ const TimeUI = {
         ) {
             const dateStaged = new Date(L_.configData.time.initialwindowend)
             if (dateStaged == 'Invalid Date') {
+                TimeUI._timelineEndTimestamp = new Date()
                 console.warn(
                     "Invalid 'Initial Window End Time' provided. Defaulting to 'now'."
                 )