parsing logs

other-logs

m1el@m1el:~/sources/log-parsing$ perl -e 'print "1409139024985\tINFO\t= Call duration 0 seconds\n" for (1..1000000)' > kid.txt
m1el@m1el:~/sources/log-parsing$ perl -MJSON -ne 'print encode_json([split /\t/, $_]), "\n"' < kid.txt > kid.json
m1el@m1el:~/sources/log-parsing$ time perl -ne '@data = split /\t/, $_' < kid.txt

real    0m5.960s
user    0m5.912s
sys     0m0.044s
m1el@m1el:~/sources/log-parsing$ time perl -MJSON -ne 'decode_json($_)' < kid.json

real    0m5.456s
user    0m5.400s
sys     0m0.052s
m1el@m1el:~/sources/log-parsing$ perl -MJSON -ne '@data=split(/\t/,$_); print encode_json({time=>$data[0],type=>$data[1],data=>$data[2]}), "\n"' < kid.txt > kid2.json
m1el@m1el:~/sources/log-parsing$ time perl -ne '@data=split(/\t/,$_); $data = {time=>$data[0],type=>$data[1],data=>$data[2]}' < kid.txt

real    0m11.384s
user    0m11.316s
sys     0m0.064s
m1el@m1el:~/sources/log-parsing$ time perl -MJSON -ne '$data = decode_json($_)' < kid2.json

real    0m7.722s
user    0m7.656s
sys     0m0.064s

parsing-apache-logs

m1el@m1el:~/sources/log-parsing$ head -100000 all-logs | perl -MJSON -MData::Dumper -MApache::Log::Parser -e 'my $parser=Apache::Log::Parser->new(fast=>1);print encode_json($parser->parse($_)||{}), "\n" for <>' > 100k.json
m1el@m1el:~/sources/log-parsing$ time head -100000 all-logs | perl -MData::Dumper -mApache::Log::Parser -e 'my $parser=Apache::Log::Parser->new(fast=>1);$parser->parse($_) for <>'

real    0m10.272s
user    0m10.092s
sys     0m0.336s
m1el@m1el:~/sources/log-parsing$ time perl -MJSON -e 'decode_json($_) for <>' < 100k.json                                                                     
real    0m3.020s
user    0m2.904s
sys     0m0.112s