mmomtchev

mmomtchev

for (let url of urls) {
  try {
    const html = await download(url);
    data[url] = parse(html);
  } catch (e) {
    console.error(e);
  }
}

mmomtchev

let p = [];
for (let url of urls) {
    p.push(download(url)
        .then((html) => {
            data[url] = parse(html);
        })
        .catch((e) => console.error(e)));
}
await Promise.allSettled(p);

mmomtchev

const Queue = require('async-await-queue');
/**
 * No more than 10 parallel, spaced at least 100ms apart
 * These are typical fair-use limitations of public APIs
 **/
const queue = new Queue(10, 100);
let p = [];
for (let url of urls) {
    /* Generate a queue ID */
    const me = Symbol();

mmomtchev

const Queue = require('async-await-queue');
/**
 * No more than 10 parallel, spaced at least 100ms apart
 * These are typical fair-use limitations of public APIs
 **/
const queue = new Queue(10, 100);
let p = [];
for (let url of urls) {
    /* Each iteration is an anonymous async function */
    p.push((async () => {

mmomtchev

const fs = require('fs');
const readline = require('readline');
const parseline = require('./parseline');

const rs = fs.createReadStream('allCountries.txt');
const rl = readline.createInterface({ input: rs });

(async () => {
	console.time(__filename);
	for await (const line of rl) {

mmomtchev

function parseline(line) {
	const data = line.split('\t');
	if (data[1] && data[1].match(/^Paris$/g))
		counter++;
}

mmomtchev

const fs = require('fs');
const readline = require('readline');
const parseline = require('./parseline');

const rs = fs.createReadStream('allCountries.txt');
const rl = readline.createInterface({ input: rs });

console.time(__filename);
rl.on('line', (line) => {
	parseline(line);

mmomtchev

const fs = require('fs');
const es = require('event-stream');
const parseline = require('./parseline');

const rs = fs.createReadStream('allCountries.txt');
console.time(__filename);
rs.pipe(es.split()).pipe(es.through(parseline));
rs.on('close', () => console.timeEnd(__filename));

mmomtchev

const fs = require('fs');
const rs = fs.createReadStream('allCountries.txt');
const parseline = require('./parseline');

console.time(__filename);
let remainder = '';
rs.on('data', (buf) => {
	const lines = (remainder + buf).split(/\r?\n/g);
	remainder = lines.pop();
	for (const line of lines) {