multiple logstash.confs

casper_thrane_url_split.conf

input {
    generator {
        type => "generated"
        message => 'http://www.elasticsearch.org/blog/series-c-financing-sharing-fruit-labors/?example=yes&kv=true&style=elk'
        count => 1
    }
}

filter {
    # If you want to chomp off the http:// URI portion:
    ruby { code => "event['message'] = event['message'].reverse.chomp('http://'.reverse).reverse" }

    # split the message
    mutate { split => ["message", "/"] }

    # add a different field name
    mutate { add_field => ["url-host", "%{message[2]}"] }
}

output {
    stdout { codec => rubydebug }
}

filezilla_patterns

MAIL \b[a-zA-Z0-9._%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b
FILEZILLA_ID \(%{BASE10NUM:fz_id:int}\)
FILEZILLA_IP %{NOTSPACE}%{IPV4:fz_ip}%{NOTSPACE}
FILEZILLA_DATESTAMP %{MONTHNUM}/%{MONTHDAY}/%{YEAR} %{HOUR}:%{MINUTE}:%{SECOND} %{WORD}
FILEZILLA_LOG %{FILEZILLA_ID} %{FILEZILLA_DATESTAMP:timestamp} - %{MAIL:mail} \(%{IPV4:fz_ip}\)> %{GREEDYDATA:logmsg}

joshmyers_conditional_outputs.conf

input {
    generator {
        type => "generated"
        message => "cool"
        tags => ["linux"]
        count => 1
    }
    generator {
        type => "generated"
        message => "uncool"
        tags => ["mac"]
        count => 1
    }
    generator {
        type => "generated"
        message => "huh"
        tags => ["windows"]
        count => 1
    }

}

filter { }

output {
    if "linux" in [tags] {
        stdout { codec => rubydebug }
        elasticsearch { host => "localhost" index => "linux" }
    }
    else if "mac" in [tags] {
        stdout { codec => json }
        elasticsearch { host => "localhost" index => "mac" }
    }
}

json_lines_example.log

{ "@timestamp": "2015-02-02T04:25:26+0000", "@fields": { "status": 200, "request": "/author/", "bytes": 43, "more": { "host" : "localhost"} } }
{ "@timestamp": "2015-02-02T04:25:27+0000", "@fields": { "status": 404, "request": "/title/", "bytes": 84, "more": { "host" : "localhost"} } }
{ "@timestamp": "2015-02-02T04:25:28+0000", "@fields": { "status": 302, "request": "/publisher/", "bytes": 45, "more": { "host" : "localhost"} } }
{ "@timestamp": "2015-02-02T04:25:29+0000", "@fields": { "status": 202, "request": "/keywords/", "bytes": 700, "more": { "host" : "localhost"} } }

kingdom_joy_json_and_json_lines_codec.conf

input {
    # http://logstash.net/docs/1.4.2/inputs/generator
    generator {
        type => "generated"
        message => '{ "@timestamp": "2014-01-01T03:14:15+0000", "@fields": { "status": 200, "request": "/quote/", "bytes": 612, "more": { "host" : "localhost"} } }'
        count => 1
        codec => json
    }
    # http://logstash.net/docs/1.4.2/inputs/exec
    exec {
        command => "cat json_lines_example.log"
        codec => json_lines
        interval => 60
    }
}

filter {
}

output {
    stdout { codec => rubydebug }
}

lognomish_kv_filter.conf

input {
    generator {
        type => "generated1"
        message => 'shost=1.1.1.1 dhost=2.2.2.2 app=internet explorer url=www.google.com'
        count => 1
    }
    generator {
        type => "generated2"
        message => 'shost=1.1.1.1 dhost=2.2.2.2 app=internet explorer url=www.google.com'
        count => 1
    }
}

filter {
    if [type] == "generated1" {
        kv {}
    }
    else if [type] == "generated2" {
        # exact same output, just making field_split and value_split explicit.
        # neither of these can catch "app=internet explorer" because it splits
        # on the space and is not a parser of any sort.
        kv { 
            field_split=" "
            value_split="="
        }
    }
}

output {
    stdout { codec => rubydebug }
}

marion-faugoin-csv-input.conf

input {
    generator {
        count => 1
        lines => [
            'id;age;name;date',
            '1;24;Ehtesh;2014/07/10',
            '5;20;Jimmy;2014/05/22',
            '1023;38;Johannes;2012/07/25',
            '5453;22;Emily;2010/02/25'
        ]
    }
}

filter {
    csv {
        add_field => [ "comment", "Parsing CSV file :D" ]
        separator => ";"
        columns => ["id", "age", "name", "date"]
    }
}

output {
    stdout { codec => rubydebug }
}

marion-faugoin-filezilla-pattern.conf

input {
    generator {
        type => "generated"
        message => "(116082) 6/12/2014 0:00:04 AM - name@address.net (207.67.115.122)> LIST"
        count => 1
    }
}

filter {
    grok {
        patterns_dir => "filezilla_patterns"
        match => { "message" => "%{FILEZILLA_LOG}" }
        add_tag => [ "filezilla" ]
        tag_on_failure => [ "ERROR : grok matching failed" ]
    }
    date {
        # http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html
        # doesn't match the one-digit MONTH but matches everything else

        # Maybe someone else can chime in on parsing the month
        # example: "6/12/2014 0:00:04 AM"
        match => [ "timestamp", "M/DD/yyyy H:mm:ss a" ]
    }
}

output {
    stdout { codec => rubydebug }
}

tim_pearson_apache.conf

input {
    generator {
        type => "generated"
        message => '127.0.0.1 - - [16/Jun/2014:06:25:16 -0400] "POST /this/is/a/sample/path.php HTTP/1.1" 200 3891 "https://www.example.com/foo/bar/qux/baz.php?f=file&active=1" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:25.0) Gecko/20100101 Firefox/25.0" on www.example.com'
        count => 1
    }
}

filter {
    grok {
        match => { "message" => "%{COMBINEDAPACHELOG} %{NOTSPACE:https} %{NOTSPACE:virtualhost}" }
    }
    date {
        locale => "en"
        match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
}

output {
    stdout { codec => rubydebug }
}