MFX logstash config

logstash.conf

input {
  syslog {
    type => syslog
    port => 514
  }

  file {
    type => otis
    path => "/job/projects/common/var/log/otis/*.log"
    #path => "/job/projects/common/var/log/otis/*dispatcher*.log"
    #sincedb_path => "/dev/null"
    start_position => beginning
    tags => ["otis"]
  }
}


filter {
  if [type] == "syslog" {
    grok {
      match => [ "message", "%{SYSLOGBASE2}" ]
      add_tag => [ "syslog", "grokked" ]
    }
  }

  if [type] == "otis" {
    grok {
      match => ["path", "([\w/]+)-(?<module>[a-z]+)_%{GREEDYDATA:logsource}\.log"]
    }
    multiline {
      negate => true
      pattern => '^(\[|Traceback|getting)'
      what => 'previous'
    }
    grok {
      match => ["message", "\[%{TIMESTAMP_ISO8601:timestamp}\] %{LOGLEVEL:level} *%{GREEDYDATA:msg}"]
    }
    date {
      match => ["timestamp", "yyyy-MM-dd HH:mm:ss,SSS"]
    }
    mutate {
      lowercase => ["level"]
    }
    mutate {
      replace => ["message", "%{msg}"]
      remove_field => "msg"
    }
    grok {
      match => ["message", "(%{HOSTNAME:node} \(%{INT:node_num}\))?(%{INT:job_num}\:%{INT:task_num})?(%{INT:job_num})?.*  +%{GREEDYDATA:msg}"]
    }
    mutate {
      replace => ["message", "%{msg}"]
      remove_field => "msg"
    }
  }
}


output {
  elasticsearch_http {
    host => "ES_HOST"
    port => "ES_PORT"
  }
}