Logstash configuration ('sudo service logstash stop' fails )

logstash.config

input {
  # used for monitoring
  exec {
    command => "/usr/bin/touch /var/run/logstash-monit/input.touch && /bin/echo OK."
    interval => 60
    type => "monitoring"
  }
  lumberjack {
    port => 55556
    ssl_certificate => "/etc/logstash/indexer01.playground.cloud.commercetools.de.crt"
    ssl_key  => "/etc/logstash/indexer01.playground.cloud.commercetools.de.key"
  }
  tcp {
    port => 55563
    type => "clb-access"
    data_timeout => 5
  }
  pipe {
    command => "tail -F /var/run/logstash-clb-access"
    type => "clb-access"
  }
  pipe {
    command => "tail -F /var/run/logstash-cdn-access"
    type => "cdn-access"
  }
}

filter {
  # --------------------------------------------------------------------------------
  # ElasticSearch
  # --------------------------------------------------------------------------------
  if [type] == "elasticsearch" {
    multiline {
      pattern => "^\[[0-9]{4}"
      what => "previous"
      negate => "true"
    }
    grok {
      match => { "message" => "\[%{TIMESTAMP_ISO8601:es_timestamp}\]\[%{LOGLEVEL:es_severity}\s*\]\[%{NOTSPACE:es_logger}\s*\] %{GREEDYDATA:es_message}(\n(?m)%{GREEDYDATA:es_trace})?" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    if !("_grokparsefailure" in [tags]) {
      date {
        match => [ "es_timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
        add_tag => [ "dated" ]
      }
      mutate {
        replace => [ "@message", "%{es_severity} - %{es_logger} - %{es_message}" ]
        add_tag => [ "replaced" ]
      }
      mutate {
        remove_field => [ "es_message", "es_timestamp" ]
        add_tag => [ "removed" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # Mongo
  # --------------------------------------------------------------------------------
  if [type] == "mongo" {
    grok {
      match => { "message" => "%{MONGO_LOG}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    if !("_grokparsefailure" in [tags]) {
      date {
        match => [ "mongo_timestamp", "MMM dd HH:mm:ss.SSS", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss.SSS", "MMM  d HH:mm:ss" ]
        add_tag => [ "dated" ]
      }
      mutate {
        replace => [ "@message", "%{mongo_logger} - %{mongo_message}" ]
        add_tag => [ "replaced" ]
      }
      mutate {
        remove_field => [ "mongo_message", "mongo_timestamp", "mongo_day" ]
        add_tag => [ "removed" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # Sphere WS
  # --------------------------------------------------------------------------------
  if [type] == "sphere" {
    if !("@message" =~ "^\s*$") {
      drop {
        add_tag => [ "dropped" ]
      }
    }
    multiline {
      pattern => "^[0-9]{4}"
      what => "previous"
      negate => "true"
    }
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:sphere_timestamp} - \[%{GREEDYDATA:sphere_thread}\] %{LOGLEVEL:sphere_severity}\s+%{NOTSPACE:sphere_logger} - %{GREEDYDATA:sphere_message}(\n(?m)%{GREEDYDATA:sphere_trace})?" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    if !("_grokparsefailure" in [tags]) {
      date {
        match => [ "sphere_timestamp", "yyyy-MM-dd HH:mm:ss,SSS" ]
        add_tag => [ "dated" ]
      }
      mutate {
        replace => [ "@message", "%{sphere_severity} - %{sphere_logger} - %{sphere_message}" ]
        add_tag => [ "replaced" ]
      }
      mutate {
        remove_field => [ "sphere_message", "sphere_timestamp" ]
        add_tag => [ "removed" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # SYSLOG
  # --------------------------------------------------------------------------------
  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{@source_host}" ]
      add_tag => [ "grokked" ]
    }
    date {
      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
      add_tag => [ "dated" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@source_host", "%{syslog_hostname}" ]
        replace => [ "@message", "%{syslog_message}" ]
        add_tag => [ "replaced" ]
      }
      mutate {
        remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
        add_tag => [ "removed" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # Apache vhost access
  # --------------------------------------------------------------------------------
  if [type] == "apache-vhost" {
    grok {
      match => { "message" => '%{HOST:http_vhost}:%{POSINT:http_port} %{IPORHOST:http_remote} %{POSINT:http_time_usec} %{USER:http_user} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_uri}(?: HTTP/%{NUMBER:http_version})?|-)" %{NUMBER:http_status} (?:%{NUMBER:http_bytes}|-) %{QS:http_referrer} %{QS:http_useragent}' }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_field => [ "received_from", "%{@source_host}" ]
      add_tag => [ "grokked" ]
    }
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss +0000" ]
      add_tag => [ "dated" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@message", "%{http_method} %{http_uri} %{http_status}" ]
        convert => [ "http_bytes", "integer" ]
        convert => [ "http_status", "integer" ]
        convert => [ "http_time_usec", "integer" ]
        add_tag => [ "replaced" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # NGinx
  # --------------------------------------------------------------------------------
  if [type] == "nginx-error" {
    grok {
      # 2013/03/18 14:00:09 [info] 6469#0: Using 32768KiB of shared memory for push module in /etc/nginx/nginx.conf:111
      match => { "message" => "%{DATESTAMP:nginx_timestamp} \[%{WORD:nginx_severity}\] %{GREEDYDATA:nginx_message}" }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    date {
      match => [ "nginx_timestamp", "yyyy/MM/dd HH:mm:ss" ]
      add_tag => [ "dated" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@message", "%{nginx_severity} %{nginx_message}" ]
        add_tag => [ "replaced" ]
      }
      mutate {
        remove_field => [ "nginx_message", "nginx_timestamp" ]
        add_tag => [ "removed" ]
      }
    }
  }
  # --------------------------------------------------------------------------------
  if [type] == "nginx-access" {
    mutate {
      replace => [ "@message", "%{http_method} %{http_uri} %{http_status}" ]
      convert => [ "http_bytes", "integer" ]
      convert => [ "http_status", "integer" ]
      convert => [ "http_time_sec", "float" ]
      add_tag => [ "replaced" ]
    }
  }

  # --------------------------------------------------------------------------------
  # Rackspace CloudLB access logs
  # --------------------------------------------------------------------------------
  if [type] == "clb-access" {
    grok {
      match => { "message" => '%{NUMBER:rs_account}_%{NUMBER:rs_lb} %{IPORHOST:http_vhost}(?::%{POSINT:http_port}|) %{IPORHOST:http_remote} - %{USER:http_user} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_uri}(?: HTTP/%{NUMBER:http_version})?|-)" (?:%{NUMBER:http_status}|-) (?:%{NUMBER:http_bytes}|-) %{QS:http_referrer} %{QS:http_useragent} (?:%{IP:rs_server}:%{POSINT:rs_port}|-)' }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss +0000" ]
      add_tag => [ "dated" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@message", "%{http_method} %{http_uri} %{http_status}" ]
        convert => [ "http_bytes", "integer" ]
        convert => [ "http_status", "integer" ]
        add_tag => [ "replaced" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # Rackspace CloudFiles access logs
  # --------------------------------------------------------------------------------
  if [type] == "cdn-access" {
    grok {
      match => { "message" => '\*\*\*%{SPHEREPRJKEY:cdn_container}/%{CDNLOGPATH:cdn_log}\*\*\*%{IPORHOST:http_remote} - %{USER:http_user} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:http_method} %{NOTSPACE:http_uri}(?: HTTP/%{NUMBER:http_version})?|-)" (?:%{NUMBER:http_status}|-) (?:%{NUMBER:http_bytes}|-) %{QS:http_referrer} %{QS:http_useragent}' }
      patterns_dir => "/etc/logstash/patterns"
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss +0000" ]
      add_tag => [ "dated" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@message", "%{http_method} %{http_uri} %{http_status}" ]
        convert => [ "http_bytes", "integer" ]
        convert => [ "http_status", "integer" ]
        add_tag => [ "replaced" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # DPKG log
  # --------------------------------------------------------------------------------
  if [type] == "dpkg" {
    grok {
      patterns_dir => "/etc/logstash/patterns"
      match => { "message" => '%{DATESTAMP:timestamp} %{DPKGACTION:pkg_action}(?: %{DPKGPKG:pkg_name})?(?: %{GREEDYDATA:pkg_version})?' }
      add_field => [ "received_at", "%{@timestamp}" ]
      add_tag => [ "grokked" ]
    }
    date {
      match => [ "timestamp", "yyyy-MM-dd HH:mm:ss" ]
      add_tag => [ "dated" ]
    }
    if !("_grokparsefailure" in [tags]) {
      mutate {
        replace => [ "@message", "%{pkg_action} - %{pkg_name} - %{pkg_version}" ]
        add_tag => [ "replaced" ]
      }
    }
  }

  # --------------------------------------------------------------------------------
  # Billing
  # --------------------------------------------------------------------------------
  if [type] == "nginx-access" {
    # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
    # API calls (nginx access.log, sphere-projects-ws)
    # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
    # Analyze/fill billing fields

    if !("http_config" == "sphere-projects-ws") {
      mutate {
        add_tag => "billing"
        add_field => [ "billing_bandwidth", "%{http_bytes}" ]
      }
    }
    if "billing" in [tags] {
      grok {
        patterns_dir => "/etc/logstash/patterns/"
        match => [ "http_uri", "^\/%{SPHEREPRJKEY:billing_prjkey}\/" ]
      }
    }
    if !("_grokparsefailure" in [tags]) {
      if "billing" in [tags] {
        # Valid billing entries (where project-key is valid)
        # Convert the fields from array to single-value (why is it a array at all?)
        mutate {
          join => [ "billing_prjkey", "" ]
          join => [ "billing_bandwidth", "" ]
        }
        mutate {
          convert => [ "billing_bandwidth", "integer" ]
        }
      }
    }
    if "billing" in [tags] and "_grokparsefailure" in [tags] {
      # Invalid billing entries (where project-key is invalid)
      # Remove "billing" tag, so record is skipped later in the pipe
      mutate {
        add_tag => [ "_billingfailure" ]
        remove_tag => [ "billing" ]
      }
    }
    # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
    # CDN usage (CDN logs)
    # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
    if !("_grokparsefailure" in [tags]) {
      mutate {
        add_field => [ "billing_prjkey", "%{cdn_container}" ]
        add_field => [ "billing_bandwidth", "%{http_bytes}" ]
        add_tag => [ "billing" ]
      }
      if "billing" in [tags] {
        mutate {
          convert => [ "billing_bandwidth", "integer" ]
        }
      }
    }
  }
}
output {
  elasticsearch {
    host => "logs01.xxx.yyy.zzz.de"
    protocol => "http"
    #cluster => "logstash"
  }
}