S3-compatible buckets file sync. Useful to backup logs and other meaningful files, when Docker's Log Driver is "json-file"

s3sync.sh

#!/bin/bash

# Functions
EscapeFilename()
{
  printf '%s' "$1" | sed -e 's/[^A-Za-z0-9._-]/_/g'
}

# Configuration
S3CMD_PARAMETERS=
BUCKET_PREFIXES=("do-space-logs/" "aws-s3bucket-logs") # Configure how many buckets you want to sync
SYNC_COMMANDS=("s3cmd sync" "aws s3 sync") # Configure command line for each bucket prefix
CONTAINERS_DIR="/var/lib/docker/containers"

# List containers
_c=$(ls ${CONTAINERS_DIR} 2>&1)
_r=$?

if [[ "${_r}" -ne "0" ]]; then
  echo "Could not list containers inside ${CONTAINERS_DIR}: ${_c}" >&2
  exit 1
fi

# Loop through all containers
_containers=$(echo "${_c}" | tr '\r\n' ' ')
for dirname in ${_containers}; do
  dir=${CONTAINERS_DIR}/${dirname}

  # Check if file is a directory
  if [[ ! -d "$dir" ]]; then
    continue
  fi

  # Get container name
  _c=$(docker inspect ${dirname} --format "{{.Name}}")
  _r=$?

  if [[ "${_r}" -ne "0" ]]; then
    echo "Could not get container name for ${dir}: ${_c}" >&2
    continue
  fi
  container_name=$(EscapeFilename $(echo ${_c} | sed -e 's/^\///g'))

  # Push logs to S3 bucket
  files=$(find ${dir} -mindepth 1 -maxdepth 1 -type f -name "*.log*")
  _r=$?

  if [[ "${_r}" -ne "0" ]]; then
    echo "Could not list files inside ${CONTAINERS_DIR}: ${_c}" >&2
    exit 1
  fi

  for raw_filename in ${files}; do
    filename=$(echo "${raw_filename}" | xargs -0 basename -a)
    fname=$(EscapeFilename "${filename}")
    fjson=${dir}/${filename}

    for ((i = 0; i < ${#BUCKET_PREFIXES[@]}; ++i)); do
      _c1="${SYNC_COMMANDS[$index]}"
      _c2="${BUCKET_PREFIXES[$index]}"
      _c3=$(head -n1 "${fjson}" | jq -r '.time' | sed 's/[-\:]//g' | sed 's/\..\+//g' | sed 's/T/_/g')
      _c4=
      if [[ "${fjson}" == *.log ]]; then
        _c4=_current
      fi
      ${_c1} $@ "${fjson}" s3://${_c2}${container_name}/${_c3}${_c4}.log ${S3CMD_PARAMETERS}
    done
  done

done

echo "S3 Sync finished"