A script in node for receiving piped web logs via tail and outputting the number of ips and requests per second

web-log-filter.js

#!/usr/bin/node
/*jslint node: true, sloppy:true */
/**
 *Receives input from tail -F /path/to/web/log | node web-log-filter.js
 *Then outputs
 *Sun Apr 08 2012 19:01:42 GMT+0000 (UTC) time since start 1190 seconds
Requests in the last 5 minutes: 874 (0.97 req/second)
IPs in the last 5 minutes: 331 (0.37 ips/second)
2 requests and 1 IPs purged
 *At most every 5 seconds
 */
var ips = [];
var requests = [];
var ipTTL = [];
var start = new Date().getTime();
var lastDisplay = 0;
var interval = 900;

process.stdin.resume();

process.stdin.on('data', function (chunk) {
    chunk += '';
    var i, purgedRequests = 0, purgedIPs = 0, str = '', ip, newipTTL = [], newRequests = [], time = new Date().getTime(), lines = chunk.split('\n');

    //cleanup
    for (i = 0; i < requests.length; i += 1) {
        if (requests[i] >= time - interval * 1000) {
            newRequests.push(requests[i]);
        }
    }

    purgedRequests = requests.length - newRequests.length;

    requests = newRequests;

    for (i = 0; i < ipTTL.length; i += 1) {
        if (ipTTL[i].ttl < time - interval * 1000) {
            delete ips[ips.indexOf(ipTTL.ip)];
        } else {
            newipTTL.push(ipTTL[i]);
        }
    }

    purgedIPs = ipTTL.length - newipTTL.length;

    ipTTL = newipTTL;

    for (i = 0; i < lines.length; i += 1) {
        ip = lines[i].match(/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/);
        if (ip === null) {
            return;
        }
        if (ips.indexOf(ip[0]) === -1) {
            ips.push(ip[0]);
            ipTTL.push({'ip' : ip[0], 'ttl' : time});
        }

        requests.push(time);

        if (lastDisplay < time - 5000) {
            str += new Date(time) + ' time since start ' + Math.floor((time - start) / 1000) + ' seconds';
            str += '\nRequests in the last 5 minutes: ' + requests.length + ' (' + (Math.round((requests.length * 100) / interval) / 100) + ' req/second)';
            str += '\n';
            str += 'IPs in the last 5 minutes: ' + ips.length + ' (' + (Math.round((ips.length * 100) / interval) / 100) + ' ips/second)';
            str += '\n';
            str += purgedRequests + ' requests and ' + purgedIPs + ' IPs purged';
            str += '\n';

            process.stdout.write(str);
            lastDisplay = time;
        }

    }
});

process.stdin.on("end", function () {
    process.stdout.write("Stream closed\n");
});