Refactoring of wmo generation

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flyingeek/lidojs",
-  "version": "1.6.67",
+  "version": "1.6.68",
   "description": "convert Lido OFP text files",
   "publishConfig": {
     "registry": "https://npm.pkg.github.com/"
@@ -19,9 +19,10 @@
     "coverage": "jest --collectCoverageFrom=src/**.js --coverage src",
     "build": "webpack",
     "build_with_wmo": "webpack && node src/makeWmo.js",
-    "makefish": "node src/makeFish.js",
-    "makewmo": "node src/makeWmo.js",
-    "updateogimet": "node src/updateOgimetIdx.js",
+    "makefish": "node scripts/makeFish.js",
+    "makewmo": "node scripts/makeWmo.js",
+    "diffogimet": "node scripts/diffOgimetIdx.js",
+    "updateogimet": "node scripts/updateOgimetIdx.js",
     "release": "npm version patch && webpack"
   },
   "browserslist": [

scripts/makeWmo.js

@@ -0,0 +1,56 @@
+const fs = require('fs/promises');
+const path = require('path');
+const Geohash = require('ngeohash');
+const {wmoParser} = require('./wmo_parsers');
+
+/* Our current list of Ogimet's known WMO index */
+const ogimetIds = require('./ogimet_idx.json');
+
+/* If needed we can exlude stations by their ID (like 74038) or their name (like LIMT) */
+const excludedStations = [];
+
+/* Where to save */
+const wmoVarPath = "./dist/wmo.var.js";
+const wmoPath = "./dist/wmo.json";
+
+console.log(`${excludedStations.length} excluded stations`);
+console.log(`${ogimetIds.length} ogimet stations`);
+console.log(`consider updating ogimet stations with npm run updateogimet`);
+
+/**
+ * geoEncode
+ * @param {Array} data [label, latitude, longitude]
+ * @param {number} precision (geohash precision)
+ * @returns {Object} indexed by geohash, a list of [[label, latitude, longitude],...]
+ */
+function geoEncode(data, precision=3) {
+  const results = {};
+  for (const [label, latitude, longitude] of data) {
+    const geohash = Geohash.encode(latitude, longitude, precision);
+    const value = [label, +latitude.toFixed(6), +longitude.toFixed(6)]
+    if (geohash in results) {
+        results[geohash].push(value);
+    } else {
+        results[geohash] = [value];
+    }
+  }
+  return results;
+}
+
+wmoParser(ogimetIds, excludedStations).then(async data => {
+  await fs.mkdir(path.dirname(wmoPath), {'recursive': true});
+  const geohashedData = geoEncode(data);
+  fs.writeFile(wmoPath, JSON.stringify(geohashedData), (err) => {
+    if (err) {
+      throw err;
+    } else {
+      console.log(`Saved ${data.length} stations!`);
+    }
+  });
+  // according to Google engineers, JSON.parse is faster than the native js parsing
+  fs.writeFile(wmoVarPath, `var WMO=JSON.parse('${JSON.stringify(geohashedData)}');\n`, (err) => {
+    if (err) {
+      throw err;
+    }
+  });
+});

scripts/ogimet_lib.js

@@ -0,0 +1,52 @@
+const Papa = require('papaparse');
+const {parserPromise} = require('./wmo_parsers');
+
+/*
+  returns date in the format YYYYMMDDHH00, the hours parameters indicates the offset in hour from now
+*/
+const asYYYYMMDDHH00 = (hours) => {
+  const ts = Date.now() + ((hours||0) * 3600000);
+  return (new Date(ts)).toISOString()
+    .replace(/[^\d]+/gu,'')
+    .slice(0,10) + "00";
+}
+
+/**
+  Process Ogimet SYNOP request and extract SYNOP IDs
+  As the request in itself returns all synops in the period, it can be pretty big.
+  There is a maximum of 200000 lines per request, 12 hours represents less than 50000 lines.
+*/
+function ogimet12HoursIndexSet(hours) {
+  const url = `https://www.ogimet.com/cgi-bin/getsynop?begin=${asYYYYMMDDHH00(hours-12)}&end=${asYYYYMMDDHH00(hours)}&lang=eng&header=yes`;
+  return parserPromise(url, data => {
+    const stationSet = new Set();
+    const parsed = Papa.parse(data);
+    parsed.data.slice(1).forEach((row) => {
+      if (row[0].match(/^\d{5}$/u)) {
+        stationSet.add(row[0]);
+      }
+    });
+    return stationSet;
+  });
+}
+
+/**
+ * Batch request of the last 48 hours synops
+ * @returns {Set} of wmo indexes
+ */
+async function ogimetIndexSet() {
+  const union = (setA, setB) => {
+    const u = new Set(setA);
+    for (let elem of setB) {
+      u.add(elem);
+    }
+    return u;
+  }
+  let stations = await ogimet12HoursIndexSet(0);
+  stations = union(stations, await ogimet12HoursIndexSet(-12));
+  stations = union(stations, await ogimet12HoursIndexSet(-24));
+  stations = union(stations, await ogimet12HoursIndexSet(-36));
+  return stations;
+}
+
+module.exports = {ogimetIndexSet};

scripts/updateOgimetIdx.js

@@ -0,0 +1,25 @@
+const fs = require("fs");
+const {ogimetIndexSet} = require('./ogimet_lib');
+const previousOgimetIds = require('./ogimet_idx.json');
+
+const outputPath = "./scripts/ogimet_idx.json";
+
+ogimetIndexSet().then(stationSet => {
+  const difference = (setA, setB) => {
+      let diff = new Set(setA);
+      for (let elem of setB) {
+        diff.delete(elem);
+      }
+      return diff;
+  }
+  const previousSet = new Set(previousOgimetIds);
+  fs.writeFile(outputPath, JSON.stringify([...stationSet]), (err) => {
+    if (err) {
+      throw err;
+    } else {
+      console.log(`Saved ${stationSet.size} stations!`);
+    }
+  });
+  console.log(`${[...difference(stationSet, previousSet)].length} stations added`);
+  console.log(`removed: ${[...difference(previousSet, stationSet)]}`);
+});

scripts/wmo_parsers.js

@@ -0,0 +1,174 @@
+const {https} = require('follow-redirects');
+const Papa = require('papaparse');
+
+/* A copy of data files are stored in this gist to avoid temporary breakdown */
+const gistURL = "https://gist.github.com/flyingeek/54caad59410a1f4641d480473ec824c3";
+
+const parserPromise = (url, fn) => new Promise((resolve, reject) => {
+  https.get(url, (response) => {
+    let data = "";
+    response.on("data", (chunk) => {
+      data += chunk;
+    });
+    response.on("error", (err) => {
+      reject(err);
+    });
+    response.on("end", () => {
+      resolve(fn(data));
+    });
+  });
+});
+
+/**
+ * Promise to wmo importer
+ * Note: nsd_bbsss.txt is not updated since 2016
+ *
+ * @returns {Promise<Array>} [{wid<string>, name?<string>, longitude<float>, latitude<float>}]
+ */
+function wmoRequest(ogimetIds, url=`${gistURL}/raw/nsd_bbsss.txt`) {
+  return parserPromise(url, data => {
+    const parsed = Papa.parse(data);
+    const results = [];
+    const wids = [];
+    const normalize = (v) => {
+      const orientation = v.slice(-1);
+      const sign = ('SW'.indexOf(orientation) >= 0) ? -1 : 1;
+      let coords = ('NEWS'.indexOf(orientation) >=0) ? v.slice(0, -1) : v;
+      coords += '-0-0'  // ensure missing seconds or minutes are 0
+      const [degrees, minutes, seconds] = coords.split('-', 3).map(parseFloat);
+      return sign * (degrees + (minutes / 60) + (seconds / 3600));
+    };
+    parsed.data.forEach((row) => {
+      if (row.length < 8) return;
+      const wid = row[0] + row[1];
+      // wid, name, lon, lat
+      wids.push(wid);
+      if (ogimetIds.indexOf(wid) >= 0) {
+        results.push({
+          wid,
+          "name": (row[2] === '----') ? '' : row[2],
+          "longitude": normalize(row[8]),
+          "latitude": normalize(row[7])
+        });
+      }
+    });
+    console.log(`${wids.length} wmo stations / ${results.length} known by ogimet`);
+    return results;
+  });
+}
+
+/**
+ * Promise to vola importer
+ * Note: vola_legacy_report.txt is not updated since 2021
+ *
+ * @returns {Promise<Array>} [{wid<string>, name?<string>, longitude<float>, latitude<float>}]
+ */
+function volaRequest(ogimetIds, url=`${gistURL}/raw/vola_legacy_report.txt`) {
+  return parserPromise(url, data => {
+    const parsed = Papa.parse(data);
+    const results = [];
+    const wids = [];
+    const normalize = (v) => {
+      const orientation = v.slice(-1);
+      const sign = ('SW'.indexOf(orientation) >= 0) ? -1 : 1;
+      let coords = ('NEWS'.indexOf(orientation) >=0) ? v.slice(0, -1) : v;
+      coords += ' 0 0'  // ensure missing seconds or minutes are 0
+      const [degrees, minutes, seconds] = coords.split(' ', 3).map(parseFloat);
+      return sign * (degrees + (minutes / 60) + (seconds / 3600));
+    };
+    parsed.data.slice(1).forEach((row) => {
+      if (row.length < 28) return;
+      const wid = row[5];
+      if (wid && wid.match(/^\d{5}$/u) && row[6] === "0") { /* some stations have subindex (1, 2...) defined in row[6] */
+        if (wid !== "94907" && wids.indexOf(wid) >= 0) { // 94907 has a knwon duplicate
+          console.log(`duplicate for ${wid}`);
+        } else {
+          wids.push(wid);
+          if (ogimetIds.indexOf(wid) >= 0) {
+            results.push({
+              wid,
+              "longitude": normalize(row[9]),
+              "latitude": normalize(row[8])
+            });
+          }
+        }
+      }
+    });
+    console.log(`${wids.length} vola stations / ${results.length} known by ogimet`);
+    return results;
+  });
+}
+
+/**
+ * Promise to oscar json importer
+ * Note: Oscar is the current updated official database of WMO
+ * URL: https://oscar.wmo.int/surface/rest/api/search/station?facilityType=landFixed&programAffiliation=GOSGeneral,RBON,GBON,RBSN,RBSNp,RBSNs,RBSNsp,RBSNst,RBSNt,ANTON,ANTONt&variable=216&variable=224&variable=227&variable=256&variable=310&variable=12000
+ *
+ * @returns {Promise<Array>} [{wid<string>, name?<string>, longitude<float>, latitude<float>}]
+ */
+function oscarRequest(ogimetIds, url=`${gistURL}/raw/oscar_wmo_stations.json`) {
+  return parserPromise(url, data => {
+    const wmo = JSON.parse(data).stationSearchResults;
+    const results = [];
+    const wids = [];
+    wmo.forEach(w => {
+      const wid = w.wigosId.split('-').pop();
+      if (
+        wid.match(/^\d{5}$/u)
+        && w.wigosId.startsWith('0-20000-0-')
+        && w.stationStatusCode === 'operational'
+        && w.stationTypeName === 'Land (fixed)'
+      ) {
+        if (wids.indexOf(wid) >= 0) {
+          console.log(`duplicate for ${wid}`);
+        } else {
+          wids.push(wid);
+          if (ogimetIds.indexOf(wid) >= 0) {
+            results.push({
+              wid,
+              "longitude": parseFloat(w.longitude),
+              "latitude": parseFloat(w.latitude)
+            });
+          }
+        }
+      }
+    });
+    console.log(`${wids.length} oscar wmo stations / ${results.length} known by ogimet`);
+    return results;
+  });
+}
+
+/**
+ * Merge importers
+ */
+async function wmoParser(ogimetIds, excludedStations) {
+  const data = [];
+
+  await Promise.all([volaRequest(ogimetIds), wmoRequest(ogimetIds), oscarRequest(ogimetIds)]).then(([volaData, wmoData, oscarData]) => {
+    const addedWmoIds = [];
+    for (const {wid, name, longitude, latitude} of wmoData) {
+        if ((excludedStations.indexOf(wid) < 0) && (excludedStations.indexOf(name) < 0)) {
+          addedWmoIds.push(wid);
+          data.push([name || wid, latitude, longitude]);
+        }
+    }
+    console.log(`wmo stations processed, total: ${data.length} WMO stations`);
+    for (const {wid, longitude, latitude} of volaData) {
+      if (excludedStations.indexOf(wid) < 0 && addedWmoIds.indexOf(wid) < 0) {
+        addedWmoIds.push(wid);
+        data.push([wid, latitude, longitude]);
+      }
+    }
+    console.log(`vola stations processed, total: ${data.length} WMO stations`);
+    for (const {wid, longitude, latitude} of oscarData) {
+      if (excludedStations.indexOf(wid) < 0 && addedWmoIds.indexOf(wid) < 0) {
+        addedWmoIds.push(wid);
+        data.push([wid, latitude, longitude]);
+      }
+    }
+    console.log(`oscar stations processed, total: ${data.length} WMO stations`);
+  });
+  return data;
+}
+
+module.exports = {wmoParser, parserPromise};