Kafka pixy kafka consumer benchmarks

README.md

This is code used to benchmark throughput using the kafka-pixy kafka client.

Setup:

curl -L https://github.com/mailgun/kafka-pixy/releases/download/v0.10.1/kafka-pixy-v0.10.1-linux-amd64.tar.gz | tar xz
cd kafka-pixy-v0.10.1-linux-amd64
./kafka-pixy --kafkaPeers "localhost:9092", --zookeeperPeers "localhost:2181"

Usage:

node server.js

Caveats:

You may need to change the path value in the opts hash if you want a consumer group that isnt called bar or a topic that isnt observations.json.

Results:

Start time:1453933649.864
Processed 15052 observations
TPS: 752.5623746085273
End time: 1453933669.865
Duration: 20.000999927520752

Start time:1453933718.617
Processed 15252 observations
TPS: 762.5237531513997
End time: 1453933738.619
Duration: 20.001999855041504

Start time:1453933863.309
Processed 15020 observations
TPS: 750.9249040005247
End time: 1453933883.311

server.js

var http = require('http');

var opts = {
    host: 'localhost',
    port: '19092',
    path: '/topics/observations.json/messages?group=bar',
    agent: false // "Error: connect EADDRNOTAVAIL" without this
}
var message_count = 0;
var start_time = Date.now() / 1000; // overwritten on real start

function end() {
    var end_time = Date.now() / 1000;
    console.log('Processed ' + message_count + ' observations');
    console.log('TPS: ' + message_count / (end_time - start_time));
    console.log('End time: ' + end_time);
    console.log('Duration: ' + (end_time - start_time));
}

function cb(res) {
    message_count++;
    make_req();
    res.on('error', function(err) {
        // Do not remove this event listener
        // Does not work properly on node 0.10 otherwise
               console.log('Error: ' + err);
    });
    res.on('data', function(data) {
        // Do not remove this event listener
        // Does not work properly on node 0.10 otherwise
    });
}

function start() {
    start_time = Date.now() / 1000;
    console.log('Start time:' + start_time);
    setTimeout(end, 20000);
    http.request(opts, cb).end();
}

// Set offset
// First req to kafka-pixy proxy returns a long polling timeout
http.request(opts, function(res) {
    res.on('end', function() {start()});
    res.on('error', function() {
        //Do not remove this event listener
        // Does not work properly on node 0.10 otherwise
               console.log('Error: ' + err);
    });
    res.on('data', function(data) {
        // Do not remove this event listenter
        // Does not work properly on node 0.10 otherwise
    });
}).end();