exceptions

{:timestamp=>"2014-05-08T12:37:23.833000-0700", :message=>"Failed to flush outgoing items", :outgoing_count=>88, :exception=>#<NameError: no method 'type' for arguments (org.jruby.RubyArray) on Java::OrgElasticsearchActionIndex::IndexRequest>, :backtrace=>["/opt/logstash/lib/logstash/outputs/elasticsearch/protocol.rb:225:in `build_request'", "/opt/logstash/lib/logstash/outputs/elasticsearch/protocol.rb:205:in `bulk'", "org/jruby/RubyArray.java:1613:in `each'", "/opt/logstash/lib/logstash/outputs/elasticsearch/protocol.rb:204:in `bulk'", "/opt/logstash/lib/logstash/outputs/elasticsearch.rb:321:in `flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.17/lib/stud/buffer.rb:219:in `buffer_flush'", "org/jruby/RubyHash.java:1339:in `each'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.17/lib/stud/buffer.rb:216:in `buffer_flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.17/lib/stud/buffer.rb:193:in `buffer_flush'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.17/lib/stud/buffer.rb:112:in `buffer_initialize'", "org/jruby/RubyKernel.java:1521:in `loop'", "/opt/logstash/vendor/bundle/jruby/1.9/gems/stud-0.0.17/lib/stud/buffer.rb:110:in `buffer_initialize'"], :level=>:warn, :file=>"stud/buffer.rb", :line=>"231"}

indexer.conf

input {
  redis {
    host => "127.0.0.1"
    type => "redis-input"
    # these settings should match the output of the agent
    data_type => "list"
    key => "logstash"

    #json events from redis
    codec => json
    tags => ["indexing"]
  }
  redis {
    host => "127.0.0.1"
    type => "redis-input"
    # these settings should match the output of the agent
    data_type => "list"
    key => "oldlogstash"

    #json events from redis
    codec => oldlogstashjson
    tags => ["indexing"]
  }
}

filter {
    if "indexing" in [tags] {
        if [type] == "aurora" {
            grok {
                match => { "message" => "USER" }
                add_field => ["type","navigate"]
            }
            grok {
                match => { "message" => "(SELECT|UPDATE|INSERT|DELETE)" }
                add_field => ["type","SQL"]
            }
            grok {
                match => { "message" => "^%{TIMESTAMP_ISO8601:log_timestamp} %{DATA:logger} %{LOGLEVEL:level} %{GREEDYDATA:log_message}" }
            }
        }
        if "log4j" in [tags] {
            grok {
                match => { "message" => "^%{TIMESTAMP_ISO8601:log_timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\]\[\] %{DATA:logger} - %{GREEDYDATA:log_message}" }
            }
        }
        if "log4net" in [tags] {
            grok {
                match => { "message" => "^%{TIMESTAMP_ISO8601:log_timestamp} \[%{DATA:thread}\] %{LOGLEVEL:level} %{DATA:logger} \[%{DATA:extra}\] - %{GREEDYDATA:log_message}" }
            }
        }
        if [type] == "apache" {
            grok {
                match => { "message" => "\[%{DAY} %{MONTH:month} %{MONTHDAY:monthday} %{TIME:time} %{YEAR:year}\] \[%{LOGLEVEL:level}\] \[%{WORD} %{IPORHOST:client}\] %{GREEDYDATA:log_message}" }
            }
            grok {
                match => { "message" => "PHP %{LOGLEVEL:php_level}" }
            }
        }
        if [type] == "apache" {
            grok {
                match => { "message" => "%{LOGLEVEL:level}: %{DATA:project}: %{DATA:command}" }
            }
        }
        if [type] == "tomcat" {
            grok {
                match => { "message" => "^%{MONTH:month} %{MONTHDAY:monthday}, %{YEAR:year} %{INT:hour}:%{MINUTE:minutes}:%{SECOND:seconds} %{WORD:halfday}" }
            }
        }
        date {
            match => ["log_timestamp","YYYY-MM-dd'T'HH:mm:ss,SSSZ","YYYY-MM-dd HH:mm:ss,SSSZ","YYYY MMM dd HH:mm:ss Z","YYYY MMM dd h:mm:ss aa Z","ISO8601"]
        }
    }
}

output {
    if "indexing" in [tags] {
        elasticsearch {
          host => "127.0.0.1"
        }
    }
}

shipper.conf

input {
    file {
        type => "apache"
        path => ["/var/log/apache2/error.log"]
    }
    file {
        type => "tomcat"
        path => ["/var/log/tomcat*/localhost.*.log"]
        codec => multiline {
            pattern => "^%{MONTH} %{MONTHDAY}, %{YEAR}"
            what => previous
            negate => true
        }
    }
}

output {
    if "indexing" not in [tags] {
      redis {
        host => "indexer.company.com"
        data_type => "list"
        key => "logstash"
      }
    }
}