Debug logstash config with ES, S3 and some handy filters

logstash.conf

input {
  http {
    port => 8080
  }
}

filter {

  if [headers][request_path] == "/liveness" {
    drop {}
  }
  
  throttle {
    after_count => 2
    period => 10
    max_age => 20
    key => "user_id"
    add_tag => "throttle_warn"
  }

  throttle {
    after_count => 3
    period => 10
    max_age => 20
    key => "user_id"
    add_tag => "throttled"
  }

  if "throttled" in [tags] {
    drop{}
  }

  if "throttle_warn" in [tags] {
    mutate {
      replace => {"message" => "throttling events for user:%{user_id}" }
    }
  }

  mutate {
    convert => {
      "user_id" => "integer"
    }
  }

  mutate {
    add_field => {"[@metadata][docker_compose]" => "${DEBUG}" }
  }

}

output {
  elasticsearch { hosts => ["${ES_HOST}:9200"] }

  stdout { }

  if [@metadata][docker_compose] != "true" {
    s3 {
      bucket => "your-logs"
      prefix => "logstash/"
      codec => "json"
      encoding => "gzip"
      access_key_id => "${AWS_ACCESS_KEY_ID}"
      secret_access_key => "${AWS_SECRET_ACCESS_KEY}"
      region => "us-east-1"
    }
  }
}