logstast-s3-input

logstast-s3-input.conf

input {
  s3 {
    bucket => "xxx"
    credentials => [ "xxx", "yyy" ]
    type => "s3-access-logs"
    prefix => "production"
    region_endpoint => "us-west-1"
    region => "us-west-1"
    sincedb_path => "/root/.sincedb_s3-access-logs"
  }
s3 {
    bucket => "xxx"
    credentials => [ "xxx", "yyy" ]
    type => "s3-access-logs-staging"
    prefix => "staging"
    region_endpoint => "us-west-1"
    region => "us-west-1"
    sincedb_path => "/root/.sincedb_s3-access-logs-staging"
  }
  s3 {
    bucket => "xxx"
    credentials => [ "xxx", "yyy" ]
    type => "elb-access-logs"
    region_endpoint => "us-west-1"
    region => "us-west-1"
    sincedb_path => "/root/.sincedb_elb-media"
  }
  s3 {
    bucket => "xxx"
    credentials => [ "xxx", "yyy" ]
    type => "elb-access-logs"
    region_endpoint => "us-west-1"
    region => "us-west-1"
    sincedb_path => "/root/.sincedb_elb-profile"
  }
}
filter {
    if [type] == "s3-access-logs" {
        grok {
            patterns_dir => ["/usr/local/logstash-1.5.0-rc3/plugins/logstash-patterns-core/patterns/"]
            match => { "message" => "%{S3_ACCESS_LOG}" }
            remove_field => ["message"]
            add_field => { "unique_view" => "%{key}_%{clientip}"}
        }
        date {
            match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
        }
    }else if [type] == "s3-access-logs-staging" {
        grok {
            patterns_dir => ["/usr/local/logstash-1.5.0-rc3/plugins/logstash-patterns-core/patterns/"]
            match => { "message" => "%{S3_ACCESS_LOG}" }
            remove_field => ["message"]
            add_field => { "unique_view" => "%{key}_%{clientip}"}
        }
        date {
            match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
        }
    }else if [type] == "elb-access-logs" {
        grok {
            patterns_dir => ["/usr/local/logstash-1.5.0-rc3/plugins/logstash-patterns-core/patterns/"]
            match => { "message" => "%{ELB_ACCESS_LOG}" }
            remove_field => ["message", "port", "httpversion", "clientport"]
        }
        date {
            match => [ "timestamp" , "ISO8601" ]
        }
    }
}
output {
    if [type] == "s3-access-logs" {
	    elasticsearch { 
	      host => "127.0.0.1"
	      protocol => "http"
	      index => "production-s3-%{+YYYY.ww}"
	    }
    }else if [type] == "s3-access-logs-staging" {
	    elasticsearch { 
	      host => "127.0.0.1"
	      protocol => "http"
	      index => "staging-s3-%{+YYYY.MM}"
	    }
    }else if [type] == "elb-access-logs" {
	    elasticsearch { 
	      host => "127.0.0.1"
	      protocol => "http"
	      index => "production-elb-%{+YYYY.MM}"
	    }
    }
}