logstash-filebeat-to-s3

gistfile1.txt

#####################
# clinet: filebeat
#####################
$cat /etc/filebeat/filebeat.yml

filebeat:
  prospectors:
    -
      paths:
        - /apphome/MetaData/log/*
      document_type: web-app
      input_type: log
      multiline:
        pattern: "^[[:digit:]]{4}"
        negate: true
        match: after
    -
      paths:
        - /var/log/httpd/intage.access_log
      document_type: http-access
      input_type: log
    -
      paths:
        - /var/log/httpd/intage.error_log
      document_type: http-error
      input_type: log
  registry_file: /var/lib/filebeat/registry
output:
  logstash:
    hosts: ["logstash-srv-01:5044"]
shipper:
logging:
  files:
  
#####################
# server: logstash
#####################
$cat /etc/logstash/conf.d/logstash-conf.json

input {
  beats {
    port => 5044
  }
}

filter {
  if [path] =~ "\/cposthome\/MetaData\/log\/" {
    grok {
      patterns_dir => ["/opt/logstash/extra_patterns"]
      match => ["message", "%{TIMESTAMP_4S:timestamp_4s} %{GREEDYDATA}"]
      add_field => ["timestamp", "%{timestamp_4s} +0900"]
    }
    date {
      match => ["timestamp", "yyyy/MM/dd HH:mm:ss Z"]
      remove_field => ["timestamp", "timestamp_4s"]
    }
  }
}

output {
  if [path] =~ "\/cposthome\/MetaData\/log\/" {
    s3 {
      access_key_id => "your_key"
      secret_access_key => "your_secret"
      region => "ap-northeast-1"
      bucket => "intage-web"
      prefix => "log/"
      codec => "json_lines"
      time_file => 1
    }
  }
  else if [path] =~ "\/var\/log\/httpd\/intage\.access_log" {
    s3 {
      access_key_id => "your_key"
      secret_access_key => "your_secret"
      region => "ap-northeast-1"
      bucket => "intage-http-access"
      prefix => "log/"
      codec => "json_lines"
      time_file => 1
    }
  }
  else if [path] =~ "\/var\/log\/httpd\/intage\.error_log" {
    s3 {
      access_key_id => "your_key"
      secret_access_key => "your_secret"
      region => "ap-northeast-1"
      bucket => "intage-http-error"
      prefix => "log/"
      codec => "json_lines"
      time_file => 1
    }
  }
}