logstash message

client_logstash_agent.conf

# Default config example from:
# - http://logstash.net/docs/1.0.14/getting-started-simple
# XXX: add platform knowledge to tag messages
input {
  file {
    path => ["/var/log/knewton/*/*.log"]
    type => "knewton_logback"
  }
  file {
    path => ["/var/log/knewton/Spiky/current", 
             "/var/log/knewton/Haagen/current"]
    type => "knewton_python_svlogd"
  }
  file {
    type => "linux-syslog"
    path => [ "/var/log/auth.log", "/var/log/messages", "/var/log/syslog" ]
  }
}

filter {
  grok {
    type => "knewton_logback"
    patterns_path => "/usr/local/share/grok/patterns"
    pattern => ["%{JAVALOGBACK}"]
    named_captures_only => false
  }

  grok {
    type => "knewton_python_svlogd"
    patterns_path => "/usr/local/share/grok/patterns"
    pattern => ["%{RUNIT}"]
    named_captures_only => false
  }

  date {
    type => "knewton_logback"
    timestamp => "yyyy-MM-dd HH:mm:ss,SSS"
  }
  date {
    type => "knewton_python_svlogd"
    runit_timestamp => "yyyy-MM-dd'T'HH:mm:ss.SSS"
  }

  date {
    type => "linux-syslog"

    # The 'timestamp' and 'timestamp8601' names are for fields in the
    # logstash event.  The 'SYSLOGLINE' grok pattern above includes a field
    # named 'timestamp' that is set to the normal syslog timestamp if it
    # exists in the event.
    timestamp => "MMM  d HH:mm:ss"   # syslog 'day' value can be space-leading
    timestamp => "MMM dd HH:mm:ss"
    timestamp8601 => ISO8601 # Some syslogs use ISO8601 time format
  }

  multiline {
    type => "knewton_logback"
    pattern => "^[0-9]{4}-[0-9]{2}-[0-9]{2} " # This goes with
    negate => true                            # this
    # pattern => "^(\s+|com|org|net|Caused)" # Doesn't work as well
    what => "previous"
  }
}

output {
  gelf {
    host => "graylog2.SERVER"
    facility => "Platform"
  }
  zeromq {
    address => "tcp://ZMQ_SERVER:2120"
    mode => "client"
  }   
}

log_message

{"@source":"file://ip-10-190-101-53//var/log/knewton/haag/haag.log","@type":"knewton_logback","@tags":[],"@fields":{"JAVALOGBACK":["2012-06-10 03:11:26,359 DEBUG"],"timestamp":["2012-06-10 03:11:26,359"],"YEAR":["2012"],"MONTHNUM":["06"],"MONTHDAY":["10"],"HOUR":["03"],"MINUTE":["11"],"SECOND":["26"],"MILLISECOND":["359"],"severity":["DEBUG"]},"@timestamp":"2012-06-10T03:11:26.359Z","@source_host":"ip-10-190-101-53","@source_path":"//var/log/knewton/haag/haag.log","@message":"2012-06-10 03:11:26,359 DEBUG [pool-14-thread-19] c.k.haag.server.event.Updater : Offer StudentBusMessage: eventId: Jun.10.2012.03:11:26.349.0:0, studentId: 7af7de2e-b295-11e1-bf57-13313d2d760a"}

server_logstash_agent.conf

input {
  zeromq {
    address => "tcp://0.0.0.0:2120"      
    # format => "json"
    format => "json_event"
    # message_format => "json"
    mode => "server"
    queue_size => 40960
    type => "zmq"
  }  
}


filter {
  mutate {
    add_field => [ "foo_source" , "%{@source}" ]
  }
}

output {
  file {
    path => "/var/tmp/zmq_logstash"
    type => "zmq"
  }
  elasticsearch {
    cluster => "logstash"
    embedded => false
    host => "SERVER"
    type => "zmq"
  }
}