cloudtrail

cloudtrail.conf:

input {
      s3 {
        'bucket' => 'bucket_name'
        'credentials' => ['access_key','secret_key']
        'delete' => false
        'interval' => '30'
        'prefix' => 'logs/'
        #'region' => 'us-east-1'
        'region_endpoint' => 'us-east-1'
        'type' => 'json'
      }
}

filter {
    if !("splitted" in [tags]) {
      json {
        'source' => 'message'
      }
      split {
        'add_tag' => ['splitted']
        'add_tag' => ['cloudtrail']
        'field' => 'Records'
      #  'refilter' => true
      #  'reuse_element' => false
      }
    }

    if ("splitted" in [tags]) {
      date {
        'match' => ['eventTime', 'ISO8601']
        'remove_tag' => ['splitted']
        'add_tag' => ['cloudtrail']
      }
      ruby {
        'code' => '["responseElements","requestParameters"].each { |field| event[field] = event[field].inspect }  '
          }
    }
}

output {
  elasticsearch {
    cluster => "logstash"
    host => "elasticsearch_server"
    embedded => "false"
    protocol => "http"
  }
}

command and stacktrace:

./bin/logstash -f cloudtrail.conf

Registering s3 input {:bucket=>"bucket_name", :region_endpoint=>"us-east-1", :level=>:info, :file=>"logstash/inputs/s3.rb", :line=>"69"}

Pipeline started {:level=>:info, :file=>"logstash/pipeline.rb", :line=>"78"}

New Elasticsearch output {:cluster=>"logstash", :host=>"elasticsearch_server", :port=>"9200", :embedded=>false, :protocol=>"http", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"252"}
Automatic template management enabled {:manage_template=>"true", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"258"}
Using mapping template {:template=>"{  \"template\" : \"logstash-*\",  \"settings\" : {    \"index.refresh_interval\" : \"5s\"  },  \"mappings\" : {    \"_default_\" : {       \"_all\" : {\"enabled\" : true},       \"dynamic_templates\" : [ {         \"string_fields\" : {           \"match\" : \"*\",           \"match_mapping_type\" : \"string\",           \"mapping\" : {             \"type\" : \"string\", \"index\" : \"analyzed\", \"omit_norms\" : true,               \"fields\" : {                 \"raw\" : {\"type\": \"string\", \"index\" : \"not_analyzed\", \"ignore_above\" : 256}               }           }         }       } ],       \"properties\" : {         \"@version\": { \"type\": \"string\", \"index\": \"not_analyzed\" },         \"geoip\"  : {           \"type\" : \"object\",             \"dynamic\": true,             \"path\": \"full\",             \"properties\" : {               \"location\" : { \"type\" : \"geo_point\" }             }         }       }    }  }}", :level=>:info, :file=>"logstash/outputs/elasticsearch.rb", :line=>"284"}