Skip to content

Instantly share code, notes, and snippets.

@rwky
Created April 8, 2012 19:03
Show Gist options
  • Save rwky/2339242 to your computer and use it in GitHub Desktop.
Save rwky/2339242 to your computer and use it in GitHub Desktop.
A script in node for receiving piped web logs via tail and outputting the number of ips and requests per second
#!/usr/bin/node
/*jslint node: true, sloppy:true */
/**
*Receives input from tail -F /path/to/web/log | node web-log-filter.js
*Then outputs
*Sun Apr 08 2012 19:01:42 GMT+0000 (UTC) time since start 1190 seconds
Requests in the last 5 minutes: 874 (0.97 req/second)
IPs in the last 5 minutes: 331 (0.37 ips/second)
2 requests and 1 IPs purged
*At most every 5 seconds
*/
var ips = [];
var requests = [];
var ipTTL = [];
var start = new Date().getTime();
var lastDisplay = 0;
var interval = 900;
process.stdin.resume();
process.stdin.on('data', function (chunk) {
chunk += '';
var i, purgedRequests = 0, purgedIPs = 0, str = '', ip, newipTTL = [], newRequests = [], time = new Date().getTime(), lines = chunk.split('\n');
//cleanup
for (i = 0; i < requests.length; i += 1) {
if (requests[i] >= time - interval * 1000) {
newRequests.push(requests[i]);
}
}
purgedRequests = requests.length - newRequests.length;
requests = newRequests;
for (i = 0; i < ipTTL.length; i += 1) {
if (ipTTL[i].ttl < time - interval * 1000) {
delete ips[ips.indexOf(ipTTL.ip)];
} else {
newipTTL.push(ipTTL[i]);
}
}
purgedIPs = ipTTL.length - newipTTL.length;
ipTTL = newipTTL;
for (i = 0; i < lines.length; i += 1) {
ip = lines[i].match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/);
if (ip === null) {
return;
}
if (ips.indexOf(ip[0]) === -1) {
ips.push(ip[0]);
ipTTL.push({'ip' : ip[0], 'ttl' : time});
}
requests.push(time);
if (lastDisplay < time - 5000) {
str += new Date(time) + ' time since start ' + Math.floor((time - start) / 1000) + ' seconds';
str += '\nRequests in the last 5 minutes: ' + requests.length + ' (' + (Math.round((requests.length * 100) / interval) / 100) + ' req/second)';
str += '\n';
str += 'IPs in the last 5 minutes: ' + ips.length + ' (' + (Math.round((ips.length * 100) / interval) / 100) + ' ips/second)';
str += '\n';
str += purgedRequests + ' requests and ' + purgedIPs + ' IPs purged';
str += '\n';
process.stdout.write(str);
lastDisplay = time;
}
}
});
process.stdin.on("end", function () {
process.stdout.write("Stream closed\n");
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment