Skip to content

Instantly share code, notes, and snippets.

@kaustavha
Created Feb 20, 2016
Embed
What would you like to do?
Kafka pixy kafka consumer benchmarks

This is code used to benchmark throughput using the kafka-pixy kafka client.

Setup:

curl -L https://github.com/mailgun/kafka-pixy/releases/download/v0.10.1/kafka-pixy-v0.10.1-linux-amd64.tar.gz | tar xz
cd kafka-pixy-v0.10.1-linux-amd64
./kafka-pixy --kafkaPeers "localhost:9092", --zookeeperPeers "localhost:2181"

Usage:

node server.js

Caveats:

You may need to change the path value in the opts hash if you want a consumer group that isnt called bar or a topic that isnt observations.json.

Results:

Start time:1453933649.864
Processed 15052 observations
TPS: 752.5623746085273
End time: 1453933669.865
Duration: 20.000999927520752
Start time:1453933718.617
Processed 15252 observations
TPS: 762.5237531513997
End time: 1453933738.619
Duration: 20.001999855041504
Start time:1453933863.309
Processed 15020 observations
TPS: 750.9249040005247
End time: 1453933883.311
var http = require('http');
var opts = {
host: 'localhost',
port: '19092',
path: '/topics/observations.json/messages?group=bar',
agent: false // "Error: connect EADDRNOTAVAIL" without this
}
var message_count = 0;
var start_time = Date.now() / 1000; // overwritten on real start
function end() {
var end_time = Date.now() / 1000;
console.log('Processed ' + message_count + ' observations');
console.log('TPS: ' + message_count / (end_time - start_time));
console.log('End time: ' + end_time);
console.log('Duration: ' + (end_time - start_time));
}
function cb(res) {
message_count++;
make_req();
res.on('error', function(err) {
// Do not remove this event listener
// Does not work properly on node 0.10 otherwise
console.log('Error: ' + err);
});
res.on('data', function(data) {
// Do not remove this event listener
// Does not work properly on node 0.10 otherwise
});
}
function start() {
start_time = Date.now() / 1000;
console.log('Start time:' + start_time);
setTimeout(end, 20000);
http.request(opts, cb).end();
}
// Set offset
// First req to kafka-pixy proxy returns a long polling timeout
http.request(opts, function(res) {
res.on('end', function() {start()});
res.on('error', function() {
//Do not remove this event listener
// Does not work properly on node 0.10 otherwise
console.log('Error: ' + err);
});
res.on('data', function(data) {
// Do not remove this event listenter
// Does not work properly on node 0.10 otherwise
});
}).end();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment