Skip to content

Instantly share code, notes, and snippets.

@arnecls
Last active August 29, 2015 14:21
Show Gist options
  • Save arnecls/80767eb87043a9a68158 to your computer and use it in GitHub Desktop.
Save arnecls/80767eb87043a9a68158 to your computer and use it in GitHub Desktop.
logstash2elastic.conf
input
{
kafka
{
zk_connect => "10.2.3.240:2181/stream/kafka/de"
codec => plain
topic_id => "mmbb"
consumer_threads => 1
reset_beginning => true
}
}
filter {
grok {
match => {message => "^(?<local_ip>[^\s]+?) (?<hostname>[^\s]+?) (?<private_ip>.*?),?\s*(?<forwarded>[^\s]+?) - (?<customdate>\d[^\s]+) \"(?<method>\w+) (?<url_path>[^\?]+)(?:\?(?<query>[^\s]*?)\s+)?HTTP\/(?<http_version>[^\s]+?)\" (?<code>\d+) (-|(?<size>\d+)) (-|(?<response_time>\d+))\s+(?<page_id>[^\s]+?)\s+(-|(?<db>\d+))\s+(-|(?<java>\d+))\s+(-|(?<memcache>\d+))\s+(-|(?<xcache>\d+))\s+(-|(?<ftp>\d+))\s+(-|(?<solr>\d+))\s+(-|(?<redis>\d+))\s+\"-?(?:(http|https):\/\/(?<referer_host>[^\/]+)\/?\??(?<referer_query>[^\s]*?))?\" \"(?<unique_ident>[^\s]+?)\" \"(?<user_agent>.+)\"$" }
}
if "_grokparsefailure" in [tags] {
drop { }
}
date {
match => ["customdate", "yyyyMMddHHmmss"]
}
mutate {
}
}
output {
elasticsearch {
protocol => "http"
host => "elastic05"
index => "accesslog"
index_type => "httplog_v5"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment