logstash_solr.conf

# Solr configuration
input {
  file {
    path => "/var/log/syslog"
    start_position => "end"
  }
}

filter {
  if [message] =~ "crawler" or [message] =~ "milnomada" {
    mutate {
      remove_field => ["@version","path","tags","status","host"]
      gsub => [ "message", "[\\\"]", "" ]

    }
  }
  else {
    drop { }
  }
  grok {
    match => {
      "message" => "%{SYSLOGTIMESTAMP} %{DATA:messagetype} %{DATA:hostname}\: %{HOSTNAME:ip} %{DATA:text} %{DATA:text} \[%{DATA:date}\] %{DATA:http_method} %{DATA:request_url} %{DATA:http_version} %{DATA:http_code} %{DATA:response_payload} %{DATA:http_referer} %{GREEDYDATA:user_agent}"
    }
  }
  mutate {
    # rename => { "@timestamp" => "id" }
    rename => { "message"=>"_text_"  }
    # add_field => { "_version_" => "-1" }
    remove_field => ["messagetype","text"]
  }
  ruby {
    code => 'event.set("id", Time.now.strftime("%Y%m%d%H%M%S%L").to_i )'
  }

}

output {
  solr_http {
    solr_url => "http://{ip}:{port}/solr/{collection}"
    flush_size => 1 # Fast (test)
  }

  stdout { codec => rubydebug }
}