logstash.conf

logstash.conf

input {
  kafka {
    bootstrap_servers => "kafka-mangkyu.mangkyu.com:9092"
    topics => ["works_mangkyu"]
    group_id => "works-mangkyu-log"
    client_id => "mangkyu-logstash"
    decorate_events => true
    consumer_threads => 10
    codec => json
  }
}

filter {
  if [message] =~ "l7check" {
      drop {}
  }

  if [fields][log_type] == "nginx" {
    grok {
      match => { "message" => '%{NOTSPACE:requestId} %{IP:ip} - - \[%{HTTPDATE:requestDate}\] "(?:%{WORD:httpMethod} %{NOTSPACE:requestURL}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawRequest})" %{NUMBER:httpStatus} %{NUMBER:bodyBytes} "%{DATA:referer}" "%{BASE16FLOAT:responseTime}" "%{DATA:userAgent}" "%{DATA:forwardedFor}" "%{DATA:tlsVersion}" "%{DATA:contentLength}" "%{GREEDYDATA:etc}"'}
    }

    date {
          match => [ "[requestDate]", "dd/MMM/yyyy:HH:mm:ss +0900" ]
          timezone => "Asia/Seoul"
          target => "@timestamp"
    }

    ruby {
       code => "event.set('date', event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
    }
  }


  if [fields][log_type] == "tomcat" {
    grok {
      match => { "message" => '\[%{NOTSPACE:requestId}\] %{TIMESTAMP_ISO8601:requestDate} \[%{LOGLEVEL:log_level}%{SPACE}*\]'}
    }

    date {
          match => [ "[requestDate]", "yyyy-MM-dd HH:mm:ss.SSS" ]
          timezone => "Asia/Seoul"
          target => "@timestamp"
    }

    ruby {
      code => "event.set('date', event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
    }
  }
}

output {
  elasticsearch {
        hosts => ["http://elastic-mangkyu.com:10200"]
        index => "%{[fields][log_type]}-%{[fields][project]}-%{[fields][region]}-%{date}"
        user => "mangkyu"
        password => "naver!23"
  }
}