Last active
June 13, 2020 10:25
-
-
Save mmomtchev/01087291701f2251e2b7a31a1f7b3a13 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Queue = require('async-await-queue'); | |
/** | |
* No more than 10 parallel, spaced at least 100ms apart | |
* These are typical fair-use limitations of public APIs | |
**/ | |
const queue = new Queue(10, 100); | |
let p = []; | |
for (let url of urls) { | |
/* Each iteration is an anonymous async function */ | |
p.push((async () => { | |
const me = Symbol(); | |
await queue.wait(me, 0); | |
try { | |
const html = await download(url); | |
data[url] = parse(html); | |
} catch (e) { | |
console.error(e); | |
} finally { | |
queue.end(me); | |
} | |
})()); | |
} | |
await Promise.allSettled(p); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment