Move files from an s3 bucket to a second buffer bucket, if the buffer bucket contains fewer than $file_count objects.

s3-buffer.sh

#!/bin/sh
#file_count=100
#source_bucket="example-log-stream"
#buffer_bucket="example-log-stream-buffer"

# Move $file_count files from an s3 bucket to a second buffer bucket, if the buffer
# bucket contains fewer than $file_count objects.
#
# We use this to prevent a logstash s3 input filter blowing up our ELK server when backlog of
# files builds up in s3 (e.g traffic spikes, or after period of system downtime.)
#
# We run this via cron */2

# If the are less than $file_count files buffered for ingestion, move to buffer bucket for logstash ingestion
[ $(s3cmd ls -r s3://$buffer_bucket | wc -l) -lt $file_count ] && \
  s3cmd ls -r --limit $file_count s3://${source_bucket} 2> /dev/null \
  | awk -v source="$source_bucket" -v buffer="$buffer_bucket" '{ \
      sub("s3://" source, "", $4); \
      system("s3cmd mv --skip-existing -q s3://" source $4 " s3://" buffer $4); \
    }'