Update ACLs, object by object, for buckets with millions of objects inside

update-s3-objects-acls-huge-buckets.sh

#!/usr/bin/env bash



## VARIABLES
BUCKET_NAME="YOUR-BUCKET-TO-PROCESS"
BUCKET_PREFIX="THE-PREFIX-TO-START-PROCESSING/"

NEXT_FILE_TO_PROCESS=""
BASE_LIST_COMMAND="aws s3api list-objects-v2 --bucket ${BUCKET_NAME} --prefix ${BUCKET_PREFIX} --output text --query 'Contents[].Key' --max-items 100 --page-size 100"
FILES_COUNT=0



## SCRIPT MAIN EXECUTION
while :
do

  # Prepare to get another page if we already iterated over files in the past
  LIST_COMMAND="$BASE_LIST_COMMAND"
  if [ "$NEXT_FILE_TO_PROCESS" != "" ]; then
    LIST_COMMAND="$BASE_LIST_COMMAND --start-after ${NEXT_FILE_TO_PROCESS}"
  fi

  # Show the command that will be evaluated
  echo "PAGINATED_FILES=\$($LIST_COMMAND)"

  # Execute crafted command
  if ! eval "PAGINATED_FILES=\$($LIST_COMMAND)"; then
    printf "The command execution has failed for this iteration\n"
  fi

  # Loop over files, one by one
  for FILE in ${PAGINATED_FILES}
  do
    # Ignore 'None' files doe to an AWS bug
    if [ "$FILE" = "None" ]; then
      echo "[INFO] 'None' file detected: skipped"
      continue
    fi

    # Ignore directories
    if [ "${FILE: -1}" = "/" ]; then
      echo "[INFO] directory detected: skipped"
      continue
    fi

    printf "[INFO] Current file to process: %s\n" "$FILE";

    # Update ACLs for the file
    aws s3api put-object-acl --bucket "${BUCKET_NAME}" --acl public-read --key "${FILE}"

    # Store the last file which was processed
    NEXT_FILE_TO_PROCESS="$FILE"
    FILES_COUNT=$(( FILES_COUNT + 1 ))
  done

  printf "Current processed files: %i \n" FILES_COUNT

done