Tagging S3 Objects by path

tags3.sh

#! /bin/bash

#
# This file will filter all s3 objects where the key/path contains a specific pattern and apply the provided tag.
# Results are split into X number of backfground processes, each tagging 1000 objects 
#


BUCKET_NAME="YOURBUCKET"
function tagObjects {
  type=$1
  pattern=$2
  tagging=$3

  # get listing, save to file
  if [ ! -e objects-${type}.txt ];then
    aws s3api list-objects-v2 \
    --bucket ${BUCKET_NAME} \
    --prefix artifacts \
    --query "Contents[?contains(Key,${pattern})].[Key]" \
    --output text > objects-$type.txt
  fi
  echo "Got objects"

  # split file if >1000 to run parallel
  mkdir splitfiles-$type
  cd splitfiles-$type
  split -l 1000 ../objects-$type.txt
  cd ..

  mkdir logs-$type
  function tag_objects {
    logpath=logs-$type/$(basename ${1}).log
    while read key;do
      echo "Tagging ${key}" >> $logpath
      aws s3api put-object-tagging --bucket ${BUCKET_NAME} --key ${key} --tagging ${tagging} >> $logpath
    done < $1
    echo "Done file $1"
  }
  for file in splitfiles-${type}/*;do
    echo "Sending file ${file} o background"
    tag_objects ${file} &
  done

  wait
}

tagObjects "cache" "'global/cache'" "TagSet=[{Key=circleci.objectType,Value=project.cache}]"
tagObjects "workspaces" "'workflows/workspaces'" "TagSet=[{Key=circleci.objectType,Value=workflow.workspace}]"
tagObjects "artifact" "'/artifact/'" "TagSet=[{Key=circleci.objectType,Value=job.artifact}]"