ES6 Generator example

download-mixpanel-people.js

'use strict';
/*
 * This gist is meant to accompany the blog post at
 * https://emgeee.com/making-multiple-ajax-requests-with-es6-generators
 */

var request = require('request')
var Promise = require('bluebird')
var fs = require('fs')
var crypto = require('crypto')

// INSERT MIXPANEL CREDENTIALS HERE 
var token = '';
var api_key = '';
var api_secret = '';

var endpoint = 'http://mixpanel.com/api/2.0/engage/?'

function encodeJson(json, encode) {
  return Object.keys(json).sort().map(function(key) {
    return key + '=' + (encode ? encodeURIComponent(json[key]) : json[key])
  })
}

function hashArgs(args) {
  var s = encodeJson(args).join('')

  var md5sum = crypto.createHash('md5')
  md5sum.update(s)
  md5sum.update(api_secret)

  return md5sum.digest('hex')
}

function getData(params) {
  var defer = Promise.defer()

  if(params.sig) {
    delete params.sig;
  }
  params.sig = hashArgs(params)
  var url_string = encodeJson(params, true).join('&')

  request.get(endpoint + url_string, function(error, res, body) {
    var data = JSON.parse(body);

    if(data.error) {
      defer.reject(data.error);
    } else {
      defer.resolve(data);
    }
  })

  return defer.promise;
}

function spawn(callback) {
  return function() {
    var iter = callback.apply(this, arguments);

    return Promise.resolve().then(function onValue(lastValue){
      var result = iter.next(lastValue);

      var done  = result.done;
      var value = result.value;

      if (done) return value; // generator done, resolve Promise
      return Promise.resolve(value).then(onValue, iter.throw.bind(iter)); // repeat
    });
  };
}

function *fetchData(params) {
  // make the inital request that will determine how many
  // more requests will need to be made
  var data = yield getData(params);

  var results = data.results;

  var global_total = data.total // total data points to be retrieved
  var page = data.page          // page number to retrieve
  var session_id = data.session_id
  params.session_id = session_id

  console.log('total', global_total)

  // continue making requests until all records have been received
  while(results.length < global_total) {
    page += 1;
    console.log(global_total - results.length + ' remaining')

    params.page = page;
    data = yield getData(params);

    results.push.apply(results, data.results);
  }

  return results;
}


var params = {
  api_key: api_key,
  expire: Math.floor(Date.now()/1000) + 60, // set session to expire 1 minute from now
};

// fetch the data!
spawn(fetchData)(params)
  .then(function(results) {
    console.log('Done!')

    // save the data to a file
    fs.writeFileSync('data.json', JSON.stringify(results, null, '  '))
  })