apache-no-io

input {
  stdin {}
}

filter {
  grok {
    match => {
      "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'
    }
  }

  geoip {
    source => clientip
    target => geoip
  }

  useragent {
    source => agent
    target => useragent
  }

  date {
    match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]
    locale => en
  }
}

output {
  stdout {
    codec => dots
  }
}

test-specs.txt

Tested on a c4.4xlarge with. Logs were downloaded from https://s3.amazonaws.com/data.elasticsearch.org/apache_logs/apache_access_logs.tar.gz
/tmp was remounted as a tmpfs filesystem to remove IO performance as a consideration.

#!/bin/sh -x
set -x LS_HEAP_SIZE=20g
#time logstash-2.2.0/bin/logstash --debug -f apache.conf -w 16 -b 2000
cat /tmp/aal | time logstash-2.2.0/bin/logstash -f apache.conf -w 12 -b 125 > dots; wc -c dots
cat /tmp/aal | time logstash-2.1.2/bin/logstash -f apache.conf -w 12 > dots; wc -c dots

cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 12 -b 125 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 4 -b 1000 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 8 -b 500 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 12 -b 250 > dots; wc -c dots

cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 4 -b 4000 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 8 -b 2000 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.2.0/bin/logstash -f apache.conf -w 12 -b 1000 > dots; wc -c dots


cat /tmp/aalshort | time logstash-2.1.2/bin/logstash -f apache.conf -w 4 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.1.2/bin/logstash -f apache.conf -w 8 > dots; wc -c dots
cat /tmp/aalshort | time logstash-2.1.2/bin/logstash -f apache.conf -w 12 > dots; wc -c dots