This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
for (let url of urls) { | |
try { | |
const html = await download(url); | |
data[url] = parse(html); | |
} catch (e) { | |
console.error(e); | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
let p = []; | |
for (let url of urls) { | |
p.push(download(url) | |
.then((html) => { | |
data[url] = parse(html); | |
}) | |
.catch((e) => console.error(e))); | |
} | |
await Promise.allSettled(p); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Queue = require('async-await-queue'); | |
/** | |
* No more than 10 parallel, spaced at least 100ms apart | |
* These are typical fair-use limitations of public APIs | |
**/ | |
const queue = new Queue(10, 100); | |
let p = []; | |
for (let url of urls) { | |
/* Generate a queue ID */ | |
const me = Symbol(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Queue = require('async-await-queue'); | |
/** | |
* No more than 10 parallel, spaced at least 100ms apart | |
* These are typical fair-use limitations of public APIs | |
**/ | |
const queue = new Queue(10, 100); | |
let p = []; | |
for (let url of urls) { | |
/* Each iteration is an anonymous async function */ | |
p.push((async () => { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const readline = require('readline'); | |
const parseline = require('./parseline'); | |
const rs = fs.createReadStream('allCountries.txt'); | |
const rl = readline.createInterface({ input: rs }); | |
(async () => { | |
console.time(__filename); | |
for await (const line of rl) { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function parseline(line) { | |
const data = line.split('\t'); | |
if (data[1] && data[1].match(/^Paris$/g)) | |
counter++; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const readline = require('readline'); | |
const parseline = require('./parseline'); | |
const rs = fs.createReadStream('allCountries.txt'); | |
const rl = readline.createInterface({ input: rs }); | |
console.time(__filename); | |
rl.on('line', (line) => { | |
parseline(line); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const es = require('event-stream'); | |
const parseline = require('./parseline'); | |
const rs = fs.createReadStream('allCountries.txt'); | |
console.time(__filename); | |
rs.pipe(es.split()).pipe(es.through(parseline)); | |
rs.on('close', () => console.timeEnd(__filename)); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const rs = fs.createReadStream('allCountries.txt'); | |
const parseline = require('./parseline'); | |
console.time(__filename); | |
let remainder = ''; | |
rs.on('data', (buf) => { | |
const lines = (remainder + buf).split(/\r?\n/g); | |
remainder = lines.pop(); | |
for (const line of lines) { |
OlderNewer