Async/await function to fetch data from multiple URLs in parallel

async.js

/* Client side, works in Chrome 55 and Firefox 52 without transpilation */
//https://blogs.msdn.microsoft.com/typescript/2016/11/08/typescript-2-1-rc-better-inference-async-functions-and-more/
async function fetchURLs() {
    try {
      // Promise.all() lets us coalesce multiple promises into a single super-promise
      var data = await Promise.all([
        /* Alternatively store each in an array */
        // var [x, y, z] = await Promise.all([
        // parse results as json; fetch data response has several reader methods available:
        //.arrayBuffer()
        //.blob()
        //.formData()
        //.json()
        //.text()
        fetch('https://jsonplaceholder.typicode.com/posts').then((response) => response.json()),// parse each response as json
        fetch('https://jsonplaceholder.typicode.com/albums').then((response) => response.json()),
        fetch('https://jsonplaceholder.typicode.com/users').then((response) => response.json())
      ]);

      for (var i of data) {
        console.log(`RESPONSE ITEM \n`);
        for (var obj of i) {
          console.log(obj);
          //logger utility method, logs output to screen
          console.log(obj);
        }
      }

    } catch (error) {
      console.log(error);
    }
  }

/* NodeJS version */
//uses the `request` package which makes working with Node's native http methods easier
const request = require('request');

var requestAsync = function(url) {
    return new Promise((resolve, reject) => {
        var req = request(url, (err, response, body) => {
            if (err) return reject(err, response, body);
            resolve(JSON.parse(body));
        });
    });
};

const urls = [
    'https://jsonplaceholder.typicode.com/posts',
    'https://jsonplaceholder.typicode.com/albums',
    'https://jsonplaceholder.typicode.com/users'
];

/* Works as of Node 7.6 */
var getParallel = async function() {
    //transform requests into Promises, await all
    try {
        var data = await Promise.all(urls.map(requestAsync));
    } catch (err) {
        console.error(err);
    }
    console.log(data);
}

getParallel();

Great this is awesome. Thank-you.

Lovely bro. Thank you great help

This helped me out a ton. Thanks.

Awesome

I wasted 1+ day trying to figure out how to do this on my own. Very awesome utility/example - thanks so much!

Glad it helped!

Thanks!! works great, question: how can i add a custom header to one of the urls? im getting this error from one of the requests (coinbase pro)

"message: User-Agent header is required."

sorry if this is a dumb question im new to node.js

thanks again

Amen for this gist! I thought it would be easy to find a parallelized http download example in node. . . boy was I wrong. This is the most useful link I've come across, thank you!

Very useful, thank you
How do i continue even if any of the url failed.. the above code running fine if all url's are accessible.. I still want to continue display even if any one of the url stuck or failed with other promised urls. Please help..

This is great, really helped me get a grip with "super" promises - thanks a lot!
Combining with the information here https://developers.google.com/web/fundamentals/primers/promises (another good resource) I've modified code above to do:
.fetch(...).then(status).then(json).then(myFunction).catch(error) (client-side code)

status, json, myFunction and error are all methods. The error() is so I can catch the error from a specific fetch() rather than wrapping the whole lot in a try-catch.

I'm very new to promises (and fetch for that matter) so may be doing it wrong but seems to work well

oh shit, thank you dude

thank you

Thanks a lot man!
You made my day :)

Glad it helps!! 👏🏼

@bschwartz757

Thank you for this, but can I ask how you would handle a situation where there is a chain dependency (i.e. if one API returns 404, then the others should not be executed) ?

I hope you will forgive me asking but I've been searching around the internet without luck for good examples.

I am working on a server-side node project, and currently I have the following syntax but its a bit messy and I'm not sure how to best clean it up (my Javascript expertise is already limited as my background is in other languages) :

    const pageMetadata = await fetch(`http://localhost:3000/api/foo`)
    if (res.status !== 200) {
        return { notFound: true };
    }
    const metadataJson = await pageMetadata.json()
    //
    const res = await fetch('http://localhost:3000/api/bar')
    if (res.status !== 200) {
        return { notFound: true };
    }
    const posts = await res.json()
    //
    // etc. etc. ... I have 3 or 4 APIs in total to call
    // but ultimately they all need to be successful, otherwise if one fails a `return { notFound: true };` should occur

Where can I put return statements to look at the data in the console?

@bschwartz757

Thank you for this, but can I ask how you would handle a situation where there is a chain dependency (i.e. if one API returns 404, then the others should not be executed) ?

I hope you will forgive me asking but I've been searching around the internet without luck for good examples.

I am working on a server-side node project, and currently I have the following syntax but its a bit messy and I'm not sure how to best clean it up (my Javascript expertise is already limited as my background is in other languages) :

    const pageMetadata = await fetch(`http://localhost:3000/api/foo`)
    if (res.status !== 200) {
        return { notFound: true };
    }
    const metadataJson = await pageMetadata.json()
    //
    const res = await fetch('http://localhost:3000/api/bar')
    if (res.status !== 200) {
        return { notFound: true };
    }
    const posts = await res.json()
    //
    // etc. etc. ... I have 3 or 4 APIs in total to call
    // but ultimately they all need to be successful, otherwise if one fails a `return { notFound: true };` should occur

Hey sorry for the delayed response. I actually mis-read your question (and forgot the particulars of how Promise.all handles rejections) at first. In fact, Promise.all handles your case just fine on its own - it rejects immediately if any one of the promises rejects. So you'd just pass your array of promises containing the requests (requestAsync in my original code snippet) to Promise.all, and it will reject the "super promise" immediately if any of the individual requests fail (if there are any remaining requests on the stack, they will not be processed).

Hope that helps, sorry again for the late response.

Where can I put return statements to look at the data in the console?

You can put console logs anywhere in the function bodies you'd normally put them, i.e.:

var requestAsync = function(url) {
    return new Promise((resolve, reject) => {
        var req = request(url, (err, response, body) => {
console.log('body: ', body)
            if (err) return reject(err, response, body);
            resolve(JSON.parse(body));
        });
    });
};

Or

var getParallel = async function() {
    //transform requests into Promises, await all
    try {
        var data = await Promise.all(urls.map(requestAsync));
console.log('data: ', data)
    } catch (err) {
        console.error(err);
    }
    console.log(data);
}

Small fix : the client side version needs:

fetchURLs()

at line 34 :)

Hello there,
I tried to use your code, but it shows me an error in console

Could you, please, take a quick look what I could mess up there?

TypeError: i is not iterable
FetchAllUrls http://127.0.0.1:5500/assets/js/script.js:35

Had a tough time trying figuring out how to asynchronously set the data of a single object whose data is given by two different get request in a forEach(async()). Your code saved the day! 👏🏾