gistfile1.txt

input {
  lumberjack {
    port => 17000
    #type => "logs"
    ssl_certificate => "/etc/logstash/logstash.crt"
    ssl_key => "/etc/logstash/logstash.key"
  }
}
filter {
  if [service] == "nginx" and [type] == "access" {
    grok {
      match => [ "message", "%{IP:client_ip} %{QS:proxy_ip} %{IP:downstream_ip} (?:%{USERNAME:auth}|-) \[%{HTTPDATE:timestamp}\] - %{NOTSPACE:cf_id} %{URIPROTO:proto} %{URIHOST:site}(:%{INT:port})? \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response} (?:%{NUMBER:bytes:int}|-) (?:%{NUMBER:request_time:float}|-) (?:%{NUMBER:upstream_request_time:float}|-) %{QS:referrer} %{QS:agent}" ]
    }
    date {
      match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
      add_tag => ["timestamp_changed"]
    }
  }
  if [service] == "nginx" and [type] == "error" {
    grok {
      match => [
        "message",
          "%{DATESTAMP:timestamp} \[%{DATA:severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{GREEDYDATA:error} client: %{IP:client_ip}, server: %{URIHOST:site}, request: \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\"(, upstream: %{QS:upstream},)?, host: \"%{URIHOST:site2}\"(, referrer: %{QS:referrer})?",
          "%{DATESTAMP:timestamp} \[%{DATA:severity}\] %{GREEDYDATA:error}",
          "%{DATESTAMP:timestamp} %{GREEDYDATA:error}"
      ]
    }
    date {
      match => [ "timestamp", "yyyy/MM/dd HH:mm:ss" ]
      add_tag => ["timestamp_changed"]
    }
  }
  if [client_ip] =~ /:[0-9a-f]*:/ {
    mutate {
      add_tag => [ "ipv6" ]
    }
  } else { 
    mutate {
      add_tag => [ "ipv4" ]
    }
  }

  if [client_ip] {
    geoip {
      source => ["client_ip"]
      target => "geoip"
    }
  }
}
filter {
  if [application] == "alice" and [type] == "exception" {
    multiline {
      pattern => "^\s*([0-9]{4}-[0-9]{2}-[0-9]{2})"
      negate => true
      what => previous
    }
    grok {
      match => [ "message", "%{TIMESTAMP_ISO8601:time}\s+(?<error>([\S\s]+))\s+\((?<file_error>(.*))\)(?<stack>(\s*#.*)*)" ]
    }
    date { 
      match => [ "time", "YYYY-MM-dd HH:mm:ss"]
      add_tag => ["timestamp_changed"]
    }
  }
}
filter {
  if [application] == "bob" and [type] == "exception" {
    multiline {
      pattern => "Exception ID"
      negate => true
      what => previous
    }
    grok {
      match => [ "message", "^%{TIMESTAMP_ISO8601:timestamp}\s+.*\s+\{Exception ID:\s+(?<exception_id>(.*))\}\s+(?<exception>(.*))\s*\|\s*%{GREEDYDATA:error}\s*(?<stack>([\s,\S]*))" ]
    }
    date {
      match => [ "timestamp", "YYYY-MM-dd HH:mm:ss"]
      add_tag => ["timestamp_changed"]
    }
  }
}
output {
  elasticsearch { 
    host => maghreb60
    port => 11000
    protocol => 'transport'
    index => "logstash-%{+YYYY.MM.dd}-%{service}-%{type}-%{environment}"
  }
  stdout { codec => 'rubydebug' }
}


log:
2015/05/18 08:27:36 [error] 14678#0: *7010174 FastCGI sent in stderr: "Primary script unknown" while reading response header from upstream, client: 122.183.234.125, server: www.xxxxx.tz, request: "GET /wp-admin/admin-ajax.php?action=revslider_ajax_action HTTP/1.0", upstream: "fastcgi://unix:/var/run/php5-fpm/default.sock:", host: "www.xxx.tz"