parse_logs.py

#!/usr/bin/env python
import json
import sys
import time
from datetime import datetime
from pathlib import Path

"""
Parse nginx logs into a JSON log event stream for CloudWatch:

$ python parse_logs.py path/to/logs > events.json
$ aws logs put-log-events \
  --log-group-name=<group_name> \
  --log-stream-name=<stream_name> \
  --log-events=file://events.json
"""

def parse_line(line):
    _date, _time, *rest = line.split(' ')
    dt = datetime.strptime(f"{_date} {_time}", "%Y/%m/%d %H:%M:%S")
    timestamp = int(time.mktime(dt.timetuple()) * 1000)
    return {"timestamp": timestamp, "message": " ".join(rest)}

for path in sys.argv[1:]:
    data = []
    for line in Path(path).open():
        data.append(parse_line(line))
    json.dump(data, sys.stdout)