Instantly share code, notes, and snippets.
Tagging S3 Objects by path
#! /bin/bash | |
# | |
# This file will filter all s3 objects where the key/path contains a specific pattern and apply the provided tag. | |
# Results are split into X number of backfground processes, each tagging 1000 objects | |
# | |
BUCKET_NAME="YOURBUCKET" | |
function tagObjects { | |
type=$1 | |
pattern=$2 | |
tagging=$3 | |
# get listing, save to file | |
if [ ! -e objects-${type}.txt ];then | |
aws s3api list-objects-v2 \ | |
--bucket ${BUCKET_NAME} \ | |
--prefix artifacts \ | |
--query "Contents[?contains(Key,${pattern})].[Key]" \ | |
--output text > objects-$type.txt | |
fi | |
echo "Got objects" | |
# split file if >1000 to run parallel | |
mkdir splitfiles-$type | |
cd splitfiles-$type | |
split -l 1000 ../objects-$type.txt | |
cd .. | |
mkdir logs-$type | |
function tag_objects { | |
logpath=logs-$type/$(basename ${1}).log | |
while read key;do | |
echo "Tagging ${key}" >> $logpath | |
aws s3api put-object-tagging --bucket ${BUCKET_NAME} --key ${key} --tagging ${tagging} >> $logpath | |
done < $1 | |
echo "Done file $1" | |
} | |
for file in splitfiles-${type}/*;do | |
echo "Sending file ${file} o background" | |
tag_objects ${file} & | |
done | |
wait | |
} | |
tagObjects "cache" "'global/cache'" "TagSet=[{Key=circleci.objectType,Value=project.cache}]" | |
tagObjects "workspaces" "'workflows/workspaces'" "TagSet=[{Key=circleci.objectType,Value=workflow.workspace}]" | |
tagObjects "artifact" "'/artifact/'" "TagSet=[{Key=circleci.objectType,Value=job.artifact}]" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment