Indexer.conf

indexer.conf

input {
  redis {
    host => "127.0.0.1"
    type => "redis-input"
    # these settings should match the output of the agent
    data_type => "list"
    key => "logstash"

    # We use json_event here since the sender is a logstash agent
    format => "json_event"
  }
}

filter {
  # Check if syslog message has PRI using grep.   If so then :
  #   strip the syslog PRI part and create facility and severity fields.
  #   the original syslog message is saved in field %{syslog_raw_message}.
  #   the extracted PRI is available in the %{syslog_pri} field.
  #
  #   You get %{syslog_facility_code} and %{syslog_severity_code} fields.
  #   You also get %{syslog_facility} and %{syslog_severity} fields if the
  #   use_labels option is set True (the default) on syslog_pri filter.
  grep {
    type => "syslog"
    match => ["@message","<\d+>"]
    add_tag => "has_pri"
    drop => false
  }
  grok {
    type => "syslog"
    tags => [ "has_pri" ]
    pattern => [ "<%{POSINT:syslog_pri}>%{SPACE}%{GREEDYDATA:message_remainder}" ]
    add_tag => "got_syslog_pri"
    add_field => [ "syslog_raw_message", "%{@message}" ]
  }
  syslog_pri {
    type => "syslog"
    tags => [ "got_syslog_pri" ]
  }
  mutate {
    type => "syslog"
    tags => [ "got_syslog_pri" ]
    replace => [ "@message", "%{message_remainder}" ]
  }
  mutate {
    # XXX must not be combined with replacement which uses same field
    type => "syslog"
    tags => [ "got_syslog_pri" ]
    remove => [ "message_remainder" ]
  }
  # strip the syslog timestamp and force event timestamp to be the same.
  # the original string is saved in field %{syslog_timestamp}.
  # the original logstash input timestamp is saved in field %{received_at}.
  grok {
    type => "syslog"
    pattern => [ "%{SYSLOGTIMESTAMP:syslog_timestamp}%{SPACE}%{GREEDYDATA:message_remainder}" ]
    add_tag => "got_syslog_timestamp"
    add_field => [ "received_at", "%{@timestamp}" ]
  }
  mutate {
    type => "syslog"
    tags => [ "got_syslog_timestamp" ]
    replace => [ "@message", "%{message_remainder}" ]
  }
  mutate {
    # XXX must not be combined with replacement which uses same field
    type => "syslog"
    tags => [ "got_syslog_timestamp" ]
    remove => [ "message_remainder" ]
  }
  date {
    type => "syslog"
    tags => [ "got_syslog_timestamp" ]
    # season to taste for your own syslog format(s)
    syslog_timestamp => [ "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
  }
  # strip the host field from the syslog line.
  # the extracted host field becomes the logstash %{@source_host} metadata
  # and is also available in the filed %{syslog_hostname}.
  # the original logstash source_host is saved in field %{logstash_source}.
  grok {
    type => "syslog"
    pattern => [ "%{SYSLOGHOST:syslog_hostname}%{SPACE}%{GREEDYDATA:message_remainder}" ]
    add_tag => "got_syslog_host"
    add_field => [ "logstash_source", "%{@source_host}" ]
  }
  mutate {
    type => "syslog"
    tags => [ "got_syslog_host" ]
    replace => [ "@source_host", "%{syslog_hostname}" ]
    replace => [ "@message", "%{message_remainder}" ]
  }
  mutate {
    # message_remainder no longer needed.
    type => "syslog"
    tags => [ "got_syslog_host" ]
    remove => [ "message_remainder" ]
  }
  

  # strip the program and optional pid field from the syslog line.
  # available in the field %{syslog_program} and %{syslog_pid}.
  grok {
    type => "syslog"
    pattern => [ "%{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:%{SPACE}%{GREEDYDATA:message_remainder}" ]
    add_tag => "got_syslog_program"
  }
  mutate {
    type => "syslog"
    tags => [ "got_syslog_program" ]
    replace => [ "@message", "%{message_remainder}" ]
  }
  mutate {
    # message_remainder no longer needed.
    type => "syslog"
    tags => [ "got_syslog_program" ]
    remove => [ "message_remainder" ]
  }
  grok {
    type => "syslog"
    tag_on_failure => "_grokparsefailure"
    pattern => [ "Failed password for( invalid user )?(%{SPACE})?%{USERNAME:user} from %{IPORHOST:host} port %{POSINT:port} %{WORD:protocol}" ]
    add_tag => [ "ssh", "grokked", "auth_failure" ]
  }
}

output {
  #stdout { debug => true debug_format => "json"}

  elasticsearch_http {
    host => "127.0.0.1"
    flush_size => "100"
    type => "syslog"
  }

  statsd {
	increment => "event"
  }
  statsd {
	tags => [ "auth_failure", "grokked", "ssh" ]
	increment => "ssh.auth_failures"
	count => [ "ssh.auth_failures.%{user}", "1" ]
  }
}