Skip to content

Instantly share code, notes, and snippets.

@rprakashg
Last active February 3, 2022 21:36
Show Gist options
  • Save rprakashg/97c3c4744b73122bc27b648f2ef1bd93 to your computer and use it in GitHub Desktop.
Save rprakashg/97c3c4744b73122bc27b648f2ef1bd93 to your computer and use it in GitHub Desktop.
bash script to create SQS messages for files added to S3 between a specific date range
#!/bin/bash
profile={replace}
s3bucket={replace}
folder={specify}
queueprefix={replace}
purge-sqs-queue() {
if [ -z "$1" ]; then
echo "need to pass queue name prefix"
return
fi
echo "getting queue url with prefix $1"
queueUrl=$(aws sqs list-queues --queue-name-prefix $1 --profile $profile | jq -r .QueueUrls[0])
if [ -z "$queueUrl" ]; then
echo "Couldn't find queue, check make sure queue exist"
else
echo "purging queue : $queueUrl"
aws sqs purge-queue --queue-url $queueUrl --profile malabs
fi
}
cat << EOF > message-template.json
{
"Records": [
{
"s3": {
"object": { "key": "" }
}
}
]
}
EOF
send-message-to-sqs() {
endPoint=$1
key=$2
if [ -z $endPoint ] || [ -z $key ]; then
echo "Queue URL and key are required"
return
fi
#message=$(cat message-template.json | jq .Records[0].s3.object.key=\"$key\" | jq 'tojson')
cat message-template.json | jq .Records[0].s3.object.key=\"$key\" > message.json
#aws sqs send-message --queue-url $endPoint --message-body ${message} --delay-seconds 0 --profile $profile
aws sqs send-message --queue-url $endPoint --message-body file://message.json --delay-seconds 0 --profile $profile
}
purge-sqs-queue $queueprefix
echo "getting list of files added to s3 for last four days"
#query list of files in s3 for a date range and loop through each item
files=($(aws s3 ls "s3://$s3bucket/$folder" --recursive --profile $profile | awk -F '<' '{split($1, d, /[- ]/)} d[2] == 8 && d[3] <= 26 && d[3] >= 22' | awk '{print $4}' | sort -r -n))
for i in "${files[@]}"
do
# do whatever on $i
echo "Key : $i"
echo "sending message to sqs queue"
send-message-to-sqs $queueUrl $i
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment