Pumps GitHubApi events into EventStore (demo)

pump.js

var githubApi = require("node-github");
var http = require("http");
var timeUntilNextEvents = 10000;
var last_created_at = new Date();
var overlapped = false;

var uuid = (function() {
  var self = {};
  var lut = []; for (var i=0; i<256; i++) { lut[i] = (i<16?'0':'')+(i).toString(16); }
  self.generate = function() {
    var d0 = Math.random()*0xffffffff|0;
    var d1 = Math.random()*0xffffffff|0;
    var d2 = Math.random()*0xffffffff|0;
    var d3 = Math.random()*0xffffffff|0;
    return lut[d0&0xff]+lut[d0>>8&0xff]+lut[d0>>16&0xff]+lut[d0>>24&0xff]+'-'+
      lut[d1&0xff]+lut[d1>>8&0xff]+'-'+lut[d1>>16&0x0f|0x40]+lut[d1>>24&0xff]+'-'+
      lut[d2&0x3f|0x80]+lut[d2>>8&0xff]+'-'+lut[d2>>16&0xff]+lut[d2>>24&0xff]+
      lut[d3&0xff]+lut[d3>>8&0xff]+lut[d3>>16&0xff]+lut[d3>>24&0xff];
  }
  return self;
})();

var updateTimers = function() {
  if(overlapped) {
    timeUntilNextEvents += 1000;
    overlapped = false;
  } else {
    timeUntilNextEvents -= 1000;
  }
};

var pushEvent = function(id, type, categoryId, metadata, data) {
  var body = JSON.stringify(data);

  var request = new http.ClientRequest({
    hostname: '127.0.0.1',
    port: 2113,
    path: '/streams/github-' + categoryId,
    method: 'POST',
    headers: {
      'Accept': 'application/json',
      'Content-Type': 'application/json',
      'Content-Length': Buffer.byteLength(body),
      'ES-EventId': id,
      'ES-EventType': type
    }
  }, function(res) {
    console.log('STATUS: ' + res.statusCode);
    console.log('HEADERS: ' + JSON.stringify(res.headers));
    res.setEncoding('utf8');
    res.on('data', function(chunk) {
      console.log('BODY: ' + chunk);
    });
  });

  request.write(body);
  request.end();
};

var processEvent = function(event) {
  if(event.created_at < last_created_at) {
    console.log('Skipping event', event.created_at);
    overlapped = true;
    return;
  }

  console.log('Processing event', event.created_at, ':', event.type);
  last_created_at = event.created_at;
  
  pushEvent(uuid.generate(), event.type, event.type, {}, event);
  //pushEvent(uuid.generate(), event.type, event.actor.id, {}, event);
};

var downloadEvents = function() {
  github.events.get(
  {
    page: 1,
    per_page: 200
  },
  function(err, res) {
    for(var i = res.length-1 ; i >= 0; i--) {
      processEvent(res[i]);
    }
    updateTimers();
  });
  
  setTimeout(downloadEvents, timeUntilNextEvents);
  console.log('Waiting ' + timeUntilNextEvents + ' until reading API again');
};

var github = new githubApi({
  version: "3.0.0",
  debug: true,
  protocol: "https",
  host: "api.github.com",
  pathPrefix: "/api/v3",
  timeout: 5000,
  headers: {
      "user-agent": "GitHub-Api-Test"
  }
});

github.authenticate({
  type: 'oauth',
  token: '<removed>'
});

downloadEvents();