-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathindex.js
58 lines (53 loc) · 1.54 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
const csv = require('csv-parser');
const fs = require('fs');
const axios = require('axios');
var data = [];
var promises=[];
var result = [];
main()
function main() {
fs.createReadStream('data.csv')
.pipe(csv())
.on('data', (row) => {
data.push(row);
})
.on('end', () => {
console.log('CSV file successfully processed');
makePromises();
});
}
function makePromises() {
for(var d=0; d<data.length; d++) {
promises.push(fetchData(data[d].referentienummer, data[d].identifier))
}
Promise.all(promises).then(function(values) {
// only find nl-links
var output = "url,referentienummer,identifier\n"
for(var v=0; v < values.length; v++) {
if(values[v].links && values[v].links.nlwiki) {
var url = values[v].links.nlwiki.url;
output += `${url},${values[v].referentienummer},${values[v].identifier}\n`;
}
}
fs.writeFile('output.csv', output, function (err, file) {
if (err) throw err;
console.log('Output saved!');
});
});
}
function fetchData(referentienummer, identifier) {
return axios
.get('https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&props=sitelinks/urls&ids='+identifier)
.then(function(response) {
let [first] = Object.keys(response.data.entities);
let sitelinks = response.data.entities[first].sitelinks;
return {
identifier: identifier,
referentienummer: referentienummer,
links: sitelinks
};
})
.catch(function(error) {
return { success: false };
});
}