parse.py

import csv
import json
import sys
from collections import defaultdict

from dateutil.parser import parse as dateparse


count = int(sys.argv[2]) if len(sys.argv) > 2 else None
seconds = defaultdict(int)

with open(sys.argv[1]) as fd:
    for lineno, line in enumerate(fd):
        if count is not None and lineno >= count:
            break
        obj = json.loads(line)
        # Normalize to the second for per-second buckets
        timestamp = dateparse(obj["@timestamp"]).replace(microsecond=0)
        seconds[timestamp] += 1
        if not (lineno % 10000):
            sys.stderr.write('.'); sys.stderr.flush()

with open('logstash.csv', 'w') as fd:
    writer = csv.writer(fd)
    for index, key in enumerate(sorted(seconds.keys())):
        writer.writerow([str(index + 1), str(seconds[key])])