Last active
September 30, 2020 04:07
-
-
Save humphd/295a18d334f200eda291b2287e22a43f to your computer and use it in GitHub Desktop.
How do I download multiple things at once?
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Imagine we have a bunch of URLs we need to check (download and check their status code) | |
const urls = [ | |
// Most of these are known to be good... | |
'https://damontui.blogspot.com/feeds/posts/default?alt=rss', | |
'https://dev.to/feed/henryzerocool/', | |
'https://danielsirkovich.blogspot.com/feeds/posts/default?alt=rss', | |
'https://abdulosd.blogspot.com/feeds/posts/default?alt=rss', | |
'http://palak-chawla.blogspot.com/feeds/posts/default?alt=rss', | |
'http://zjjiang2.blogspot.com/feeds/posts/default/-/categorylabel?alt=rss', | |
'https://dev.to/feed/phast184', | |
'https://hyunjijanelee.blogspot.com/feeds/posts/default/-/Open-Source?alt=rss', | |
'https://osdnathanp.wordpress.com/feed/', | |
'https://medium.com/feed/@random4900', | |
// But this one will fail | |
'http://tea.cesaroliveira.net/archives/tag/seneca/feed', | |
]; | |
// We need to download these, let's use node-fetch (`npm install node-fetch`) | |
const fetch = require('node-fetch'); | |
// fetch() needs a `url`, and it will return a Promise. So we need to | |
// transform our Array-of-URLs into an Array-of-Promises. Let's first | |
// write a function that takes a URL, and returns a Promise that will | |
// get resolved with an Object containing the URL and its Status Code. | |
// We'll use async/await syntax, and specify that our function is `async`, | |
// meaning that it "returns a Promise" and that we'll be "using the await keyword." | |
async function checkUrl(url) { | |
// We need try/catch to deal with fetch()'s Promise erroring | |
try { | |
// We will download this resource, and await the response. To show | |
// what's happening, we'll log that we're starting | |
console.log(`Starting download for ${url}`); | |
const res = await fetch(url); | |
// If we get here, it worked, and we'll resolve with the url + status | |
return { url, status: res.status }; | |
} catch(err) { | |
// If we get here, we encountered an error, so we'll send back -1 to indicate an error | |
return { url, status: -1 }; | |
} finally { | |
// We're done processing this URL, log it so you can see things finish | |
console.log(`Finished download of ${url}`); | |
} | |
} | |
// Transform our Array-of-URLs to an Array-of-Promises, and start them all. | |
const promises = urls.map(checkUrl); | |
// We'd like to wait on *all* of these fetch() Promises to complete, so | |
// we'll use Promise.all() to get a single Promise from our Array-of-Promises | |
Promise | |
.all(promises) | |
.then(results => console.table(results)) | |
.catch(err => console.error(err)); | |
/** | |
$ node index.js | |
Starting download for https://damontui.blogspot.com/feeds/posts/default?alt=rss | |
Starting download for https://dev.to/feed/henryzerocool/ | |
Starting download for https://danielsirkovich.blogspot.com/feeds/posts/default?alt=rss | |
Starting download for https://abdulosd.blogspot.com/feeds/posts/default?alt=rss | |
Starting download for http://palak-chawla.blogspot.com/feeds/posts/default?alt=rss | |
Starting download for http://zjjiang2.blogspot.com/feeds/posts/default/-/categorylabel?alt=rss | |
Starting download for https://dev.to/feed/phast184 | |
Starting download for https://hyunjijanelee.blogspot.com/feeds/posts/default/-/Open-Source?alt=rss | |
Starting download for https://osdnathanp.wordpress.com/feed/ | |
Starting download for https://medium.com/feed/@random4900 | |
Starting download for http://tea.cesaroliveira.net/archives/tag/seneca/feed | |
Finished download of http://tea.cesaroliveira.net/archives/tag/seneca/feed | |
Finished download of https://dev.to/feed/henryzerocool/ | |
Finished download of https://dev.to/feed/phast184 | |
Finished download of https://osdnathanp.wordpress.com/feed/ | |
Finished download of http://zjjiang2.blogspot.com/feeds/posts/default/-/categorylabel?alt=rss | |
Finished download of https://damontui.blogspot.com/feeds/posts/default?alt=rss | |
Finished download of https://danielsirkovich.blogspot.com/feeds/posts/default?alt=rss | |
Finished download of https://hyunjijanelee.blogspot.com/feeds/posts/default/-/Open-Source?alt=rss | |
Finished download of https://medium.com/feed/@random4900 | |
Finished download of https://abdulosd.blogspot.com/feeds/posts/default?alt=rss | |
Finished download of http://palak-chawla.blogspot.com/feeds/posts/default?alt=rss | |
┌─────────┬────────────────────────────────────────────────────────────────────────────────┬────────┐ | |
│ (index) │ url │ status │ | |
├─────────┼────────────────────────────────────────────────────────────────────────────────┼────────┤ | |
│ 0 │ 'https://damontui.blogspot.com/feeds/posts/default?alt=rss' │ 200 │ | |
│ 1 │ 'https://dev.to/feed/henryzerocool/' │ 200 │ | |
│ 2 │ 'https://danielsirkovich.blogspot.com/feeds/posts/default?alt=rss' │ 200 │ | |
│ 3 │ 'https://abdulosd.blogspot.com/feeds/posts/default?alt=rss' │ 200 │ | |
│ 4 │ 'http://palak-chawla.blogspot.com/feeds/posts/default?alt=rss' │ 200 │ | |
│ 5 │ 'http://zjjiang2.blogspot.com/feeds/posts/default/-/categorylabel?alt=rss' │ 200 │ | |
│ 6 │ 'https://dev.to/feed/phast184' │ 200 │ | |
│ 7 │ 'https://hyunjijanelee.blogspot.com/feeds/posts/default/-/Open-Source?alt=rss' │ 200 │ | |
│ 8 │ 'https://osdnathanp.wordpress.com/feed/' │ 200 │ | |
│ 9 │ 'https://medium.com/feed/@random4900' │ 200 │ | |
│ 10 │ 'http://tea.cesaroliveira.net/archives/tag/seneca/feed' │ -1 │ | |
└─────────┴────────────────────────────────────────────────────────────────────────────────┴────────┘ | |
**/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment