HTTP log replayer

filter_logs.conf

input {
  stdin {}
}

filter {
  grok {
    match => { "message" => [ '%{IP:ip}" %{GREEDYDATA:a}', 
                              '%{IP:ip1}, %{IP:ip}" %{GREEDYDATA:a}' ] }
  }

  fingerprint {
    method => "IPV4_NETWORK"
    source => "ip"
    target => "ip"
    key => "20"
  }

  if [ip1] {
    mutate {
      add_field => { "masked" => '"%{ip1}, %{ip}" %{a}' }
    }
  } else {
    mutate {
      add_field => { "masked" => '"%{ip}" %{a}' }
    }
  }

  geoip {
    source => "ip"
  }

}

output {
  if [geoip][continent_code] == "NA" or [geoip][continent_code] == "SA" {
    file {
      path => "./access_us.log"
      codec => line { format => "%{masked}"}
    }
  } else {
    file {
      path => "./access_row.log"
      codec => line { format => "%{masked}"}
    }
  }
}

replay_file.py

#!/usr/bin/env python

import re
import time
import argparse
from datetime import datetime

def getKey(item):
    return item[0]

def sort_tuple_list(list):
    return sorted(list, key=getKey)

def timestamp_to_seconds_since_midnight(hh, mi, ss):
    return (3600 * int(hh)) + (60 * int(mi)) + int(ss)

def build_data_map(data_file_path):
    p1 = re.compile("^(.+)\[\d{2}.\w{3}.\d{4}:(.+) .+\](.*)$")
    p2 = re.compile("(\d{2}):(\d{2}):(\d{2})")

    m = {}
    l = []

    recs = [line.rstrip('\n') for line in open(data_file_path)]

    for rec in recs:
        res1 = p1.search(rec)
        res2 = p2.search(res1.group(2))
        secs = timestamp_to_seconds_since_midnight(res2.group(1), res2.group(2), res2.group(3))
        
        if secs in m:
            m[secs].append((res1.group(1), res1.group(3)))
        else:
            m[secs] = [(res1.group(1), res1.group(3))]

    return m

def get_current_utc_datetime():
    return datetime.utcnow()

def get_current_utc_timestamp(dt):
    month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
    
    return "%02d/%s/%04d:%02d:%02d:%02d +0000" % (dt.day, month[dt.month], dt.year, dt.hour, dt.minute, dt.second)

def get_offset_from_midnight(dt):
    return (3600 * dt.hour) + (60 * dt.minute) + dt.second

def output_record(rec_str):
    print(rec_str)

# Parse command line parameters
parser = argparse.ArgumentParser()

parser.add_argument('-f', help="Data file path", required=True)

args = parser.parse_args()

rec_map = build_data_map(args.f)
offset = 0

# Loop and output records until interrupted
while (True):
    time.sleep(0.5)

    cdt = get_current_utc_datetime()
    last_offset = offset
    offset = get_offset_from_midnight(cdt)
    ts = get_current_utc_timestamp(cdt)

    if offset > last_offset and rec_map[offset]:
        rec_list = rec_map[offset]

        for rec in rec_list:
            output_record("%s[%s]%s" % (rec[0], ts, rec[1]))