Skip to content

Instantly share code, notes, and snippets.

@tobsn
Created May 3, 2011 12:09
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tobsn/953219 to your computer and use it in GitHub Desktop.
Save tobsn/953219 to your computer and use it in GitHub Desktop.
node.js pipe for lighttpd logs to mongodb
#!/usr/local/bin/node
var fs = require('fs'),
useragent = require('/www/utils/node_modules/useragent'),
zoneinfo = require('/www/utils/node_modules/zoneinfo'),
TZDate = zoneinfo.TZDate,
countrycodes = require('/www/utils/node_modules/zoneinfo/lib/countrycodes');
mongo = require('/www/utils/node_modules/mongoq'),
db = mongo('mongodb:\/\/localhost/logs'),
access = db.collection( 'access' );
process.stdin.resume();
process.stdin.setEncoding( 'utf8' );
process.stdin.on( 'data', function( chunk ) {
chunk = chunk.trim().split('**#**');
var ua = useragent.parser( chunk[8] );
var req = chunk[4].split(' ');
var t = new TZDate();
t.setTimezone( 'America/Los_Angeles' );
// todo: array of stupid requests
if( req[1] != '/favicon.ico' ) {
access.insert([{
'r':((/\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/.test(chunk[0]))?ip2long(chunk[0]):chunk[0]), // remote host
'h':chunk[1], // host
't':t._date, // timestamp
'm':((req[0]=='GET')?0:1), // method 0 = get, 1 = post
'u':req[1], // url
's':parseInt(chunk[5]), // status
'b':parseInt(chunk[6]), // size
'f':((chunk[7]=='-')?0:chunk[7]), // referer
'a':ua.pretty(), // browser
'o':ua.prettyOs() // platform
}]);
}
});
process.stdin.on( 'end', function() {
db.close();
});
function ip2long(a){var b=0,a=a.match(/^([1-9]\d*|0[0-7]*|0x[\da-f]+)(?:\.([1-9]\d*|0[0-7]*|0x[\da-f]+))?(?:\.([1-9]\d*|0[0-7]*|0x[\da-f]+))?(?:\.([1-9]\d*|0[0-7]*|0x[\da-f]+))?$/i);if(!a)return!1;a[0]=0;for(b=1;b<5;b+=1)a[0]+=!!(a[b]||"").length,a[b]=parseInt(a[b])||0;a.push(256,256,256,256);a[4+a[0]]*=Math.pow(256,4-a[0]);if(a[1]>=a[5]||a[2]>=a[6]||a[3]>=a[7]||a[4]>=a[8])return!1;return a[1]*(a[0]===1||16777216)+a[2]*(a[0]<=2||65536)+a[3]*(a[0]<=3||256)+a[4]*1};
// lighttpd config: accesslog.filename = "|/path/to/log.js"
// accesslog.format = "%h**#**%V**#**%u**#**%t**#**%r**#**%>s**#**%b**#**%{Referer}i**#**%{User-Agent}i"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment