Instantly share code, notes, and snippets.

Embed
What would you like to do?
Logstash config that sends sampled traffic to Honeycomb and all traffic to stdout
input { stdin { } }
filter {
grok {
match => ["message", "%{HAPROXYHTTP}"]
remove_field => ["message"]
}
mutate {
convert => { "actconn" => "integer"
"backend_queue" => "integer"
"beconn" => "integer"
"bytes_read" => "integer"
"feconn" => "integer"
"retries" => "integer"
"srv_queue" => "integer"
"srvconn" => "integer"
"time_backend_connect" => "integer"
"time_backend_response" => "integer"
"time_duration" => "integer"
"time_queue" => "integer"
"time_request" => "integer"
"http_status_code" => "integer"
}
}
# double the stream so we can sample only traffic to honeycomb
clone {
clones => ["honeycomb"]
}
# only sample the honeycomb stream
if ([type] == "honeycomb") {
if [http_status_code] { # don't crash if http_status_code doesn't exist
if [http_status_code] >= 200 and [http_status_code] < 300 {
drop { percentage => 90 }
mutate { add_field => { "samplerate" => "10" } }
} else if [http_status_code] >= 500 {
mutate { add_field => { "samplerate" => "1" } }
} else {
drop { percentage => 50 }
mutate { add_field => { "samplerate" => "2" } }
}
}
}
}
output {
# send the sampled stream to Honeycomb
if [type] == "honeycomb" {
http {
url => "https://api.honeycomb.io/1/events/logstash_test"
http_method => "post"
headers => {
"X-Honeycomb-Team" => "xxxxx"
"X-Honeycomb-Event-Time" => "%{@timestamp}"
"X-Honeycomb-Samplerate" => "%{samplerate}"
}
format => "json"
workers => 10
}
} else {
# and send the full stream to stdout
stdout { }
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment