A proof-of-concept parser for Heroku's Postgres metrics logs written in Ruby using the Parslet gem.

README.md

Backstory

While tailing the logs for a Rails app running on Heroku, I noticed some funky entries which showed up. While these frightened me at first, some fine Herokoid data engineers assured me that my PostgreSQL database hadn't suffered trauma but was showing off the Heroku Postgres metrics logs feature.

To aid in the digestion of this neato data, I've taken the liberty of creating a Ruby parser using the Parslet gem which can convert a line of log output to an AST suitable for export to JSON.

Right now I got a little cute with parsing the timestamp but other than that, I have a real interest in getting feedback for this. Comments and suggestions welcome.

heroku_postgres_metrics_log_parser.rb

require 'parslet'

class HerokuPostgresMetricLogParser < Parslet::Parser
  rule(:colon)   { str(':') }
  rule(:digit)   { match['0-9'] }
  rule(:equals)  { str('=') }
  rule(:hyphen)  { match('-') }
  rule(:plus)    { str('+') }
  rule(:space)   { match('\s') }
  rule(:t)       { str('T') }

  rule(:year) { digit.repeat(4) }
  rule(:double_digit) { digit.repeat(2) }
  rule(:date) { year.as(:year) >> hyphen >> double_digit.as(:month) >> hyphen >> double_digit.as(:day) }


  rule(:time) { double_digit.as(:hour) >> colon >> double_digit.as(:minute) >> colon >> double_digit.as(:second) }
  rule(:offset) { double_digit.as(:hour) >> colon >> double_digit.as(:minute) }
  rule(:timestamp) { date.as(:date) >> t >> time.as(:time) >> plus >> offset.as(:offset) }

  rule(:service_designator) { str('app[heroku-postgres]') }

  rule(:preamble) { timestamp.as(:timestamp) >> space >> service_designator.as(:service_designator) }

  rule(:atomic) { match['^\s='].repeat }

  rule(:key_val_pair) { atomic.as(:key) >> equals >> atomic.as(:val) >> space.maybe }

  rule(:measurements) { key_val_pair.repeat }

  rule(:log_line) { preamble.as(:preamble) >> colon >> space >> measurements.as(:measurements) }

  root :log_line
end

output.json

{
  "preamble": {
    "timestamp": {
      "date": {
        "year": "2013",
        "month": "05",
        "day": "08"
      },
      "time": {
        "hour": "13",
        "minute": "11",
        "second": "19"
      },
      "offset": {
        "hour": "00",
        "minute": "00"
      }
    },
    "service_designator": "app[heroku-postgres]"
  },
  "measurements": [
    {
      "key": "source",
      "val": "HEROKU_POSTGRESQL_COLOR"
    },
    {
      "key": "measure.current_transaction",
      "val": "50757"
    },
    {
      "key": "measure.db_size",
      "val": "22099064bytes"
    },
    {
      "key": "measure.tables",
      "val": "29"
    },
    {
      "key": "measure.active-connections",
      "val": "3"
    },
    {
      "key": "measure.waiting-connections",
      "val": "0"
    },
    {
      "key": "measure.index-cache-hit-rate",
      "val": "0.99962"
    },
    {
      "key": "measure.table-cache-hit-rate",
      "val": "0.99998"
    }
  ]
}

parser_usage.rb

require 'json'

def parse(str)
  parser = HerokuPostgresMetricLogParser.new

  parser.parse(str)
rescue Parslet::ParseFailed => failure
  puts failure.cause.ascii_tree
end

parsed = parse("2013-05-08T13:11:19+00:00 app[heroku-postgres]: source=HEROKU_POSTGRESQL_COLOR measure.current_transaction=50757 measure.db_size=22099064bytes measure.tables=29 measure.active-connections=3 measure.waiting-connections=0 measure.index-cache-hit-rate=0.99962 measure.table-cache-hit-rate=0.99998")

puts JSON.pretty_generate(parsed)