Skip to content

Instantly share code, notes, and snippets.

@rhoml
Forked from gtrummell/mesos.conf
Created September 6, 2016 22:41
Show Gist options
  • Save rhoml/215f5848e7e47199fae75d3a231c7bf1 to your computer and use it in GitHub Desktop.
Save rhoml/215f5848e7e47199fae75d3a231c7bf1 to your computer and use it in GitHub Desktop.
Logstash configuration for Mesos
# Logstash Configuration for Mesos Masters and Slaves
input {
file {
path => "/var/log/mesos/mesos-master.[EIFW]*"
tags => [
"master"
]
}
file {
path => "/var/log/mesos/mesos-slave.[EIFW]*"
tags => [
"slave"
]
}
file {
path => "/tmp/mesos/slaves/*/frameworks/*/executors/*/runs/latest/std*"
tags => [
"slave"
]
}
}
filter {
grok {
match => {
"message" => "(?<log_severity>[EFIW])%{MONTHNUM:month}%{MONTHDAY:day}\s+%{TIME:time}\s+%{BASE10NUM:thread_id}\s+%{NOTSPACE:file}\:%{BASE10NUM:line}\]\s+%{GREEDYDATA:log_data}"
}
add_field => {
"date_time" => "2015-%{month}-%{day}T%{time}Z"
}
}
grok {
match => {
"path" => "/%{GREEDYDATA}/executors/%{GREEDYDATA:app}[.]%{GREEDYDATA}"
}
add_field => {
"date_time" => "%{@timestamp}"
}
}
if [log_severity] =~ "E" {
mutate {
replace => {
"log_severity" => "ERROR"
}
}
}
if [log_severity] =~ "F" {
mutate {
replace => {
"log_severity" => "FATAL"
}
}
}
if [log_severity] =~ "I" {
mutate {
replace => {
"log_severity" => "INFO"
}
}
}
if [log_severity] =~ "W" {
mutate {
replace => {
"log_severity" => "WARNING"
}
}
}
mutate {
convert => {
"day" => "integer"
}
}
mutate {
convert => {
"month" => "integer"
}
}
mutate {
convert => {
"line" => "integer"
}
}
mutate {
convert => {
"thread_id" => "integer"
}
}
date {
match => [
"date_time", "ISO8601"
]
}
}
output {
elasticsearch {
"codec" => plain
"host" => [
"search-logstash-production-2agp3zffh45exrxsdddpvf4g3u.us-west-2.es.amazonaws.com"
]
"index" => "mesos-production-%{+YYYYMMdd}"
"protocol" => "http"
"port" => "443"
"ssl" => "true"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment