Merge pull request data-govt-nz#4 from ebuckley/webapp-ify-script

Webapp ify script
HorizonsRC · Feb 7, 2018 · eb102e4 · eb102e4
2 parents 355ad07 + 436f77a
commit eb102e4
Show file tree

Hide file tree

Showing 8 changed files with 637 additions and 106 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+node_modules
+.vscode
diff --git a/convert.js b/convert.js
@@ -1,8 +1,8 @@
 // Small node conversion app to convert CSV open data stocktakes and convert to data.json harvestable endpoints
 
 //use the csvtojson parser as it is nice and fast https://www.npmjs.com/package/csvtojson
-const csv = require('csvtojson')
 const fs = require('fs');
+const converter = require('./convert_logic');
 
 //get arguments and flags set
 // --file path to csv file
@@ -33,72 +33,13 @@ if(process.argv.indexOf("--output") != -1){ //does our flag exist?
 }
 
 
-const converter = csv({
-  ignoreEmpty: true,
-  colParser: {
-    'keywords': function (item, head, resultRow, row, colIdx){
-       resultRow['keyword']=item.toLowerCase().replace(/(~|`|!|@|#|$|%|^|&|\*|\(|\)|{|}|\[|\]|;|:|\"|'|<|\.|>|\?|\/|\\|\||-|_|\+|=)/g,"").split(/\s*[,]\s*/);
-    },
-    'updateFrequency': function (item, head, resultRow, row , colIdx){
-      //See https://project-open-data.cio.gov/iso8601_guidance#accrualperiodicity
-      switch (item) {
-        case 'Annual':
-        case 'Annually':
-            resultRow['accrualPeriodicity']='R/P1Y'
-          break;
-        case '6-Monthly':
-            resultRow['accrualPeriodicity']='R/P6M'
-          break;
-        case 'Quarterly':
-            resultRow['accrualPeriodicity']='R/P3M'
-          break;
-        case 'Monthly':
-            resultRow['accrualPeriodicity']='R/P1M'
-          break;
-        case 'Weekly':
-            resultRow['accrualPeriodicity']='R/P1W'
-          break;
-        case 'Daily':
-            resultRow['accrualPeriodicity']='R/P1D'
-          break;
-        default:
-            resultRow['accrualPeriodicity']='irregular'
-      }
-    },
-    'issued': function (item, head, resultRow, row, colIdx){
-         return item.split("/").reverse().join("-");
-    },
-    'modified': function (item, head, resultRow, row, colIdx){
-         return item.split("/").reverse().join("-");
-    },
-    'licence': function (item, head, resultRow, row, colIdx){
-         resultRow['license']=item
-    }
-  }
-})
-.fromFile(file, function(err,result){
-    // if an error has occured then handle it
-    if(err){
-        console.log("An Error Has Occured");
-        console.log(err);
-    }
-    // create a variable called json and store
-    // the result of the conversion
-    var dataset = JSON.stringify(result,null,4);
-    var catalog =
-`{
-  "@context": "https://www.data.govt.nz/catalog.jsonld",
-	"@id": "${url}/data.json",
-	"@type": "dcat:Catalog",
-	"conformsTo": "https://www.data.govt.nz/toolkit/schema",
-  "dataset":`;
-
-    fs.writeFile(output + "data.json", catalog + dataset + '}', 'utf8', function (err) {
+const inputCSV = fs.readFileSync(file, 'utf8');
+converter.convert(inputCSV, url)
+  .then(data => {
+    fs.writeFile(output + "data.json", data, 'utf8', function (err) {
       if (err) {
           return console.log(err);
       }
       console.log("data.json generated.");
-
     });
-
-});
+  });
diff --git a/convert_logic.js b/convert_logic.js
@@ -0,0 +1,76 @@
+const csv = require('csvtojson')
+const fs = require('fs');
+
+module.exports = {}
+
+/**
+ * Take the csv content and convert it to a 
+ * @return Promise<string> dcatJson content
+ */
+module.exports.convert = (csvContent, sourceURL) => {
+
+  return new Promise((resolve, reject) => {
+    const converter = csv({
+      ignoreEmpty: true,
+      colParser: {
+        'keywords': function (item, head, resultRow, row, colIdx){
+          resultRow['keyword']=item.toLowerCase().replace(/(~|`|!|@|#|$|%|^|&|\*|\(|\)|{|}|\[|\]|;|:|\"|'|<|\.|>|\?|\/|\\|\||-|_|\+|=)/g,"").split(/\s*[,]\s*/);
+        },
+        'updateFrequency': function (item, head, resultRow, row , colIdx){
+          //See https://project-open-data.cio.gov/iso8601_guidance#accrualperiodicity
+          switch (item) {
+            case 'Annual':
+            case 'Annually':
+                resultRow['accrualPeriodicity']='R/P1Y'
+              break;
+            case '6-Monthly':
+                resultRow['accrualPeriodicity']='R/P6M'
+              break;
+            case 'Quarterly':
+                resultRow['accrualPeriodicity']='R/P3M'
+              break;
+            case 'Monthly':
+                resultRow['accrualPeriodicity']='R/P1M'
+              break;
+            case 'Weekly':
+                resultRow['accrualPeriodicity']='R/P1W'
+              break;
+            case 'Daily':
+                resultRow['accrualPeriodicity']='R/P1D'
+              break;
+            default:
+                resultRow['accrualPeriodicity']='irregular'
+          }
+        },
+        'issued': function (item, head, resultRow, row, colIdx){
+            return item.split("/").reverse().join("-");
+        },
+        'modified': function (item, head, resultRow, row, colIdx){
+            return item.split("/").reverse().join("-");
+        },
+        'licence': function (item, head, resultRow, row, colIdx){
+            resultRow['license']=item
+        }
+      }
+    })
+    .fromString(csvContent, (err, result) => {
+        // if an error has occured then handle it
+        if(err){
+          console.error('Fatal error from csv parse lib', err);
+          return reject(err)
+        }
+        // create a variable called json and store
+        // the result of the conversion
+        var dataset = JSON.stringify(result,null,4);
+        var catalog =
+    `{
+      "@context": "https://www.data.govt.nz/catalog.jsonld",
+      "@id": "${sourceURL}/data.json",
+      "@type": "dcat:Catalog",
+      "conformsTo": "https://www.data.govt.nz/toolkit/schema",
+      "dataset":`;
+        const allContent = catalog + dataset + '}';
+        resolve(allContent);
+    });
+  });
+}