-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathindex.js
82 lines (68 loc) · 2.09 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
const { ILert } = require("ilert");
const Papa = require("papaparse");
const fs = require("fs");
require("dotenv").config();
if(!process.env.ILERT_API_KEY) {
throw new Error("It seems your api key is missing!");
}
const ilert = new ILert({
apiKey: process.env.ILERT_API_KEY
});
const fetchResourcePage = (resource, index, maxResults) => {
return ilert.call("GET", null, `/${resource}`, { "start-index": index, "max-results": maxResults });
}
const fetchResource = async (resource, pageSize = 25, timeoutSec = 180) => {
// little safety net
const _t = setTimeout(() => {
console.log("timeout", timeoutSec, "reached, aborting..");
process.exit(1);
}, timeoutSec * 1000);
let resources = [];
let index = 0;
while(true) {
const page = (await fetchResourcePage(resource, index, pageSize)).data;
index = index + pageSize;
resources = resources.concat(page);
if(!page || !page.length || page.length < pageSize) {
break;
}
}
clearTimeout(_t);
return resources;
}
const writeCSV = async (resource, items) => {
const csv = Papa.unparse(items);
console.log("writing file...", csv.length);
const file = `./${resource}-${Date.now()}.csv`;
fs.writeFileSync(file, csv);
console.log(file, "written.");
}
const main = async (resource) => {
try {
console.log("fetching", resource, "hold on...");
const resources = await fetchResource(resource, 25, 180);
console.log("fetched", resources.length, resource);
await writeCSV(resource, resources);
} catch(error) {
console.log(error.message);
}
}
//Arguments syntax resource={resource}
try {
const args = process.argv
let resource;
args.map((value) => {
if(value.startsWith("resource")) {
const temp = value.split("=");
resource = temp[1];
}
});
if(resource) {
resource = resource.replace('"', '');
main(resource);
} else {
throw new Error("Not enought arguments given");
}
} catch (error) {
console.error(error);
}