Sync contents of an S3 bucket with a local folder, skipping local files with the same filename

s3-filename-sync.sh

#!/usr/bin/env bash
#
# Experimental shell script for downloading files from S3 and
# skipping local files with the same filenames.
#
# Example usage: bash shell.sh --bucket-name "my-bucket" --target-dir "./my-local-dir" --prefix "my-prefix/" --flatten "true"

set -euo pipefail
IFS=$'\n\t'

sync() {
  local bucket_name prefix target_dir flatten
  bucket_name="$1"
  prefix="$2"
  target_dir="$3"
  flatten="$4"
  
  read -r -d '' -a keys < <(aws s3api list-objects-v2 \
    --bucket "$bucket_name" \
    --prefix "$prefix" \
    --query "Contents[*].Key" \
    --output text \
    && printf '\0'
  )
  
  for key in "${keys[@]}"; do
    test "$key" = "None" && continue
    if [ "$flatten" = "true" ]; then
      target_file="$target_dir/$(basename "$key")"
    else
      target_file="$target_dir/$key"
    fi
    if [ ! -e "$target_file" ]; then
      mkdir -p "$(dirname "$target_file")"
      aws s3 cp "s3://$bucket_name/$key" "$target_file"
    fi
  done
}

main() {
  local bucket_name prefix target_dir flatten
  prefix=""
  flatten="false"
  while [ "${#:-}" -gt 0 ]; do
    case "$1" in
      --bucket-name)
        shift
        bucket_name="$1"
        shift
        ;;
      --prefix)
        shift
        prefix="$1"
        shift
        ;;
      --target-dir)
        shift
        target_dir="$1"
        shift
        ;;
      --flatten)
        shift
        flatten="$1"
        shift
        ;;
      *)
        printf "Unknown option '%s'.\n\n" "$1"; exit ;;
    esac
  done
  sync "$bucket_name" "$prefix" "$target_dir" "$flatten"
}

main "$@"