Skip to content

Instantly share code, notes, and snippets.

@eddiewebb eddiewebb/tags3.sh
Last active Jun 21, 2018

Embed
What would you like to do?
Tagging S3 Objects by path
#! /bin/bash
#
# This file will filter all s3 objects where the key/path contains a specific pattern and apply the provided tag.
# Results are split into X number of backfground processes, each tagging 1000 objects
#
BUCKET_NAME="YOURBUCKET"
function tagObjects {
type=$1
pattern=$2
tagging=$3
# get listing, save to file
if [ ! -e objects-${type}.txt ];then
aws s3api list-objects-v2 \
--bucket ${BUCKET_NAME} \
--prefix artifacts \
--query "Contents[?contains(Key,${pattern})].[Key]" \
--output text > objects-$type.txt
fi
echo "Got objects"
# split file if >1000 to run parallel
mkdir splitfiles-$type
cd splitfiles-$type
split -l 1000 ../objects-$type.txt
cd ..
mkdir logs-$type
function tag_objects {
logpath=logs-$type/$(basename ${1}).log
while read key;do
echo "Tagging ${key}" >> $logpath
aws s3api put-object-tagging --bucket ${BUCKET_NAME} --key ${key} --tagging ${tagging} >> $logpath
done < $1
echo "Done file $1"
}
for file in splitfiles-${type}/*;do
echo "Sending file ${file} o background"
tag_objects ${file} &
done
wait
}
tagObjects "cache" "'global/cache'" "TagSet=[{Key=circleci.objectType,Value=project.cache}]"
tagObjects "workspaces" "'workflows/workspaces'" "TagSet=[{Key=circleci.objectType,Value=workflow.workspace}]"
tagObjects "artifact" "'/artifact/'" "TagSet=[{Key=circleci.objectType,Value=job.artifact}]"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.