Sketch for G1 to stream lots of data from one place (twilio) to disk

limited_streamed_data.js

var accountSid = 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'; // Your Account SID from www.twilio.com/console
var authToken = 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'; // Your Auth Token from www.twilio.com/console
var client = require('twilio')(accountSid, authToken);
const fs = require('fs');
const path = require('path');
const axios = require('axios');
const async = require('async');
var tasks = [];

const BASE_URL = 'https://api.twilio.com/';
const LIMIT_CONCURRENT = 100;
const LIMIT_RECORDINGS = Infinity;

client.recordings.each({
  // The `done` callback is executed after all recordings have been processed
  done: () => {
    // The `async.cargo` function returns a `Cargo` object (see https://caolan.github.io/async/docs.html#cargo)
    // This cargo object accepts a worker, which is a function that will process tasks. 
    // But the cargo object can limit the number of tasks that are processed at the same time (using `LIMIT_CONCURRENT`)
    async
      .cargo(
        // We have defined the worker as a function that executes tasks in parallel
        (ts, callback) => async.parallel(ts, callback), 
        // This argument is the maximum limit of tasks that are processed at the same time
        // This means only `LIMIT_CONCURRENT` tasks are passed to the worker
        LIMIT_CONCURRENT
      )
      // We push our list of tasks onto the cargo object for processing
      // See https://caolan.github.io/async/docs.html#CargoObject
      .push(tasks);
  },
  // The maximum number of recordings that are to be returned and processed
  limit: LIMIT_RECORDINGS,
}, (recording) => {
  const FILENAME = recording.uri.split('/').slice(-1)[0].replace('.json', '.wav')
  const FILEPATH = path.join(__dirname, FILENAME);
  const URI = `${BASE_URL}${recording.uri.replace('.json', '')}`;

  tasks.push((callback) => {
    console.log(`STREAMING ${FILENAME}`);
    return axios.get(URI, {
      responseType: 'stream',
      auth: {
        'user': 'ACxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
        'pass': 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz',
      }
    })
      .then((response) => {
        response
          .data
          .on('error', (err) => {
            console.log('STREAM ERROR', err);
            callback(err)
          })
          .on('end', () => {
            console.log(`DONE ${FILENAME}`);
            callback()
          })
          .pipe(fs.createWriteStream(FILEPATH));
      })
      .catch((err) => {
        console.log('AXIOS ERROR', err);
        callback(err);
      });
  });
});