Scalyr RDS Postgresql log parser

gistfile1.txt

{
  lineGroupers: [
    {
      // groups line used to display stats
      start: "^[^\\s]",
      continueThrough: "^[\\s]+!"
    },
    {
      // try to group lines that do have queries with text data containing newlines. Over simplification.
      start: "^\\d+-\\d+-\\d+\\s+\\d+:\\d+:\\d+ UTC",
      continueThrough: "^[\\s]+"
    }
  ],
  
  patterns: {
    // timestamp pattern as logged: "2014-04-10 02:14:35 UTC"
    ts_pattern: "\\d+-\\d+-\\d+\\s+\\d+:\\d+:\\d+ UTC",
    
    // default prefix used by RDS is "%t:%r:%u@%d:[%p]:"
    // this pattern is not used anywhere and is for reference only
    prefix_pattern: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:",
      
    stats_pattern: "!\\s+system usage stats:\\s+!.+$"
  },
  formats: [
    {
      id: "statement",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:STATEMENT: $query{parse=sqlToSignature}$",
      halt: true
    },
    {
      id: "statement_log",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+statement: $query{parse=sqlToSignature}$",
      halt: true
    },
    {
      // enabled via log_duration or log_min_duration_statement
      // we correlate on the process_id since session id cannot be logged as of this writing
      // 2014-04-10 02:14:35 UTC:hostname(34025):user@database:[32414]:LOG:  duration: 6.399 ms
      id: "duration",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+duration: $duration_ms$ ms",
      halt: true
    },
    {
      // enabled via log_connections
      id: "connection_received",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+connection received: host=$connection_host$ port=$connection_port$",
      halt: true
    },
    {
      // enabled via log_connections
      id: "connection_authorized",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+connection authorized: user=$connection_user$ database=$connection_database$",
      halt: true
    },
    {
      // enabled via log_disconnections
      // session_time is formatted as H:mm:SS.sss for example 0:06:36.161
      id: "disconnection",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+disconnection: session time:$session_time$ user=$connection_user$ database=$connection_database$ host=$connection_host$ port=$connection_port$",
      halt: true
    },
    {
      // enabled via log_parser_stats, log_planner_stats, log_statement_stats
      // this log line is the will appear before the stats details (below)
      id: "statistics_start",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+$stats_type$ STATISTICS",
      association: {tag: "statistics", keys: ["process_id"], store: ["stats_type"]}
    },
    {
      // statistics details - see above
      id: "statistics_end",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+$stats=stats_pattern$",
      association: {tag: "statistics", keys: ["process_id"], fetch: ["stats_type"]},
      halt: true
    },
    {
      // all the session related logs which haven't been mapped above
      id: "session",
      format: "$timestamp=ts_pattern$:$remote_hostname$:$user$@$db_name$:\\[$process_id$\\]:$severity$:\\s+$details$",
      halt: true
    },
    {
      // this is basically maintenance operations such as checkpoint
      // typically:
      // 2014-04-10 01:32:38 UTC::@:[23929]:LOG:  checkpoint starting: time
      id: "default",
      format: "$timestamp=ts_pattern$::@:\\[$process_id$\\]:$severity$: $details$"
    }
  ]
}