Parse rails logs and output json ready for logstash

readlogs.cr

require "json"

class Request
  property timestamp
  property method
  property path
  property ip
  property pid
  property controller
  property action
  property format
  property params
  property status
  property statusmsg
  property totaltime
  property othertimes
  property error
  property redirect

  def initialize
    @othertimes = {} of String => Float
  end

  def to_s(io)
    io.json_object do |obj|
      obj.field "@timestamp", timestamp
      obj.field "@method", method
      obj.field "@path", path
      obj.field "@ip", ip
      obj.field "@pid", pid
      obj.field "@controller", controller
      obj.field "@action", action
      obj.field "@format", format
      obj.field "@params", params
      obj.field "@status", status
      obj.field "@statusmsg", statusmsg
      obj.field "@totaltime", totaltime
      obj.field "@othertimes", othertimes
      obj.field "@error", error
      obj.field "@redirect", redirect
    end
  end
end

processes = {} of Int32 => Request

def match(name)
  $~.not_nil![name]
end

while line = gets
  next if line == "\n"

  if line =~ /., \[(?<timestamp>\S+) \#(?<pid>\d+)\]\s*(?<level>\S+) -- :\s*(?<message>.*)/
    timestamp = match("timestamp")
    pid = match("pid").to_i
    level = match("level")
    message = match("message")

    if level == "FATAL"
      error = String.build do |str|
        while err = gets != "\n"
          str << err
        end
      end

      if req = processes[pid]?
        req.error = error
        processes.delete(pid)
      end

      next
    end

    case message
    when /Started (?<method>\S+) \"(?<path>\S+)\" for (?<ip>\S+)/
      req = processes[pid] = Request.new
      req.pid = pid
      req.timestamp = timestamp
      req.method = match("method")
      req.path = match("path")
      req.ip = match("ip")
      req.pid = pid

    when /Processing by (?<controller>\S+)\#(?<action>\S+) as (?<format>\S+)/
      req = processes[pid]
      req.controller = match("controller")
      req.action = match("action")
      req.format = match("format")

    when /Parameters: (?<params>.*)/
      req = processes[pid]
      params = match("params").replace(Regex.new("=>"), ":").replace(Regex.new(":nil"), ":null")
      req.params = Json.parse(params)

    when /Rendered/, /Connecting to database/, /Filter chain halted/
      # Ignore

    when /Redirected to (?<redirect>.*)/
      req = processes[pid]
      req.redirect = match("redirect")

    when /Completed (?<status>\d+) (?<statusmsg>.+) in (?<totaltime>[\d\.]+)ms( \((?<othertimes>.*)\))?/
      req = processes[pid]
      req.status = match("status").to_i
      req.statusmsg = match("statusmsg")
      req.totaltime = match("totaltime").to_f

      unless (othertimes = match("othertimes")).empty?
        othertimes.split(" | ").each do |othertime|
          othertime =~ /(?<name>.+): (?<value>[\d\.]+)ms/
          req.othertimes[match("name").downcase] = match("value").to_f
        end
      end

      processes.delete(pid)
      puts req
      # req.to_s(STDOUT)

    else
      STDERR.puts "Unsupported message: #{line}"
    end
  else
    STDERR.puts "Unknown line: #{line}"
  end
end