Skip to content

Instantly share code, notes, and snippets.

@ianbattersby

ianbattersby/pump.js

Created Nov 25, 2015
Embed
What would you like to do?
Pumps GitHubApi events into EventStore (demo)
var githubApi = require("node-github");
var http = require("http");
var timeUntilNextEvents = 10000;
var last_created_at = new Date();
var overlapped = false;
var uuid = (function() {
var self = {};
var lut = []; for (var i=0; i<256; i++) { lut[i] = (i<16?'0':'')+(i).toString(16); }
self.generate = function() {
var d0 = Math.random()*0xffffffff|0;
var d1 = Math.random()*0xffffffff|0;
var d2 = Math.random()*0xffffffff|0;
var d3 = Math.random()*0xffffffff|0;
return lut[d0&0xff]+lut[d0>>8&0xff]+lut[d0>>16&0xff]+lut[d0>>24&0xff]+'-'+
lut[d1&0xff]+lut[d1>>8&0xff]+'-'+lut[d1>>16&0x0f|0x40]+lut[d1>>24&0xff]+'-'+
lut[d2&0x3f|0x80]+lut[d2>>8&0xff]+'-'+lut[d2>>16&0xff]+lut[d2>>24&0xff]+
lut[d3&0xff]+lut[d3>>8&0xff]+lut[d3>>16&0xff]+lut[d3>>24&0xff];
}
return self;
})();
var updateTimers = function() {
if(overlapped) {
timeUntilNextEvents += 1000;
overlapped = false;
} else {
timeUntilNextEvents -= 1000;
}
};
var pushEvent = function(id, type, categoryId, metadata, data) {
var body = JSON.stringify(data);
var request = new http.ClientRequest({
hostname: '127.0.0.1',
port: 2113,
path: '/streams/github-' + categoryId,
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(body),
'ES-EventId': id,
'ES-EventType': type
}
}, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function(chunk) {
console.log('BODY: ' + chunk);
});
});
request.write(body);
request.end();
};
var processEvent = function(event) {
if(event.created_at < last_created_at) {
console.log('Skipping event', event.created_at);
overlapped = true;
return;
}
console.log('Processing event', event.created_at, ':', event.type);
last_created_at = event.created_at;
pushEvent(uuid.generate(), event.type, event.type, {}, event);
//pushEvent(uuid.generate(), event.type, event.actor.id, {}, event);
};
var downloadEvents = function() {
github.events.get(
{
page: 1,
per_page: 200
},
function(err, res) {
for(var i = res.length-1 ; i >= 0; i--) {
processEvent(res[i]);
}
updateTimers();
});
setTimeout(downloadEvents, timeUntilNextEvents);
console.log('Waiting ' + timeUntilNextEvents + ' until reading API again');
};
var github = new githubApi({
version: "3.0.0",
debug: true,
protocol: "https",
host: "api.github.com",
pathPrefix: "/api/v3",
timeout: 5000,
headers: {
"user-agent": "GitHub-Api-Test"
}
});
github.authenticate({
type: 'oauth',
token: '<removed>'
});
downloadEvents();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment