logstash.conf

input {
  tcp {
    port => 1514
    type => syslog
  }
  udp {
    port => 1514
    type => syslog
  }
  tcp {
    port => 5140
    type => syslog
  }
  udp {
    port => 5140
    type => syslog
  }
}

filter {
  if [type] == "syslog" {
    grok {
      patterns_dir   => [ "/etc/logstash/patterns" ]
      overwrite => "message"
      match => {
        "message" => "^(?:<%{POSINT:syslog_pri}>)?%{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}"
      }
    }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss"]
    }
    syslog_pri {}
    mutate {
      replace => ["host", "%{syslog_host}"]
      remove_field => ["syslog_host", "syslog_timestamp"]
    }
  }

  if [program] == "apache-access" {
    grok {
      patterns_dir   => [ "/etc/logstash/patterns" ]
      match => {
        "message" => "^%{IPORHOSTORDASH:vhost} %{IPORHOSTORDASH:client_ip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:accept_date}\] \"(?:%{WORD:http_verb} %{NOTSPACE:http_request}(?: HTTP/%{NUMBER:http_version})?|%{DATA:rawrequest})\" %{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-) %{QS:referrer} %{QS:useragent}"
      }
    }
    mutate {
      replace => [ "type", "apache-access" ]
      convert => [ "http_status_code", "integer" ]
      convert => [ "bytes_read", "integer" ]
      uppercase => [ "http_verb" ]
    }
  } else if [program] == "apache-error" {
    mutate {
      replace => [ "type", "apache-error" ]
    }
    grok {
      patterns_dir   => [ "/etc/logstash/patterns" ]
      match => {
        "message" => [
          "^\[%{APACHEERRORDATE:accept_date}\] \[error\] \[client %{IP:client_ip}\]"
        ]
      }
    }
    if !("_grokparsefailure" in [tags]) {
      grok {
        patterns_dir   => [ "/etc/logstash/patterns" ]
        break_on_match => "false"
        tag_on_failure => "false"
        match => {
          message => [
            "in %{PATH:absolute_path} on line %{POSINT:line_number}",
            "exist: %{PATH:absolute_path}"
          ]
        }
      }
      if [absolute_path] {
        grok {
          patterns_dir   => [ "/etc/logstash/patterns" ]
          tag_on_failure => "false"
          match => {
            absolute_path => [
              "^/mnt/%{PATHCOMPONENT:cluster}/%{PATHCOMPONENT:store}/%{PATHCOMPONENT:domain}/%{PATHCOMPONENT:instance}",
              "^/var/www/%{PATHCOMPONENT:cluster}/%{PATHCOMPONENT:domain}/%{PATHCOMPONENT:instance}"
            ]
          }
        }
        if [domain] and [instance] {
          mutate {
            add_field => [ "vhost", "%{instance}.%{domain}" ]
            remove_field => [ "instance", "cluster", "store" ]
          }
        }
      }
    }
  } else if [program] == "haproxy" {
    grok {
      patterns_dir   => [ "/etc/logstash/patterns" ]
      match => {
        "message" => "^%{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?\"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?\""
      }
    }
    mutate {
      replace => [ "type", "haproxy" ]
      convert => [ "time_request", "integer" ]
      convert => [ "time_queue", "integer" ]
      convert => [ "time_backend_connect", "integer" ]
      convert => [ "time_backend_response", "integer" ]
      convert => [ "time_duration", "integer" ]
      convert => [ "http_status_code", "integer" ]
      convert => [ "bytes_read", "integer" ]
      uppercase => [ "http_verb" ]
    }
  } else if [program] == "nannybot" {
    mutate {
      replace => ["type", "nannybot"]
    }
  } else if [program] == "rails-delayed_jobs" {
    mutate {
      replace => ["type", "delayed_jobs"]
    }
  }
}

output {
  file {
    path => "/var/log/aggregate/%{type}.%{+yyyy-MM-dd}"
  }
  gelf {
    host => "ops-log1-dnslb.sjc.sugarcrm.pvt"
    port => "12201"
  }
}