Created
September 25, 2019 03:41
-
-
Save robvanoostenrijk/6112a8c6e08a382bfcfc9b19b0ca8022 to your computer and use it in GitHub Desktop.
Remove 0 byte placeholders from AWS S3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
BUCKET_NAME="<bucket>" | |
BUCKET_PATH="<path>/" | |
PROFILE="<profile>" | |
### S3 Bulk Delete by File Size ### | |
aws s3 ls "s3://${BUCKET_NAME}/${BUCKET_PATH}" --profile "${PROFILE}" --recursive | awk -F ' ' '{print $3,$4}' | awk -F ' ' '$1 < 1 {print $2}' | xargs -IP echo '{"Key": "P"}' > delete.txt | |
# Because bulk delete limit is 1000 per api call. | |
split -l 1000 delete.txt | |
# Append json start and end parts to complete a bulk delete request in every file. | |
for file in x*; do | |
echo '{"Objects": [' >> delete"$file" && paste -d, -s "$file" >> delete"$file" && | |
echo '],"Quiet": true }' >> delete"$file" | |
done | |
# Send delete requests as json content in delete* files. | |
for file in deletex*; do | |
aws s3api delete-objects --bucket "${BUCKET_NAME}" --delete file://"$file" --profile "${PROFILE}" | |
done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment