AWS CLI scan DynamoDB table with rate limiting

ddbtablescan.sh

#!/bin/bash

if [ "$#" -ne 6 ]; then
  echo "Usage: ddbtablescan.sh <profile> <table> <attributeToFilterOn> <attributeValueToFilterWith> <maxItemsPerSec> <listOfAttributesToReturn>"
  echo "e.g."
  echo "ddbtablescan.sh my-profile Users company Amazon 10 firstName,lastName,email"
 exit 1
fi

found=$(which aws)
if [ -z "$found" ]; then
  echo "Please install the AWS CLI under your PATH: http://aws.amazon.com/cli/"
  exit 1
fi

found=$(which jq)
if [ -z "$found" ]; then
  echo "Please install jq  under your PATH: https://github.com/stedolan/jq/"
  exit 1
fi

q_mid=\'\\\'\'

NEXTTOKEN="null"
PROFILE=$1
TABLE=$2
FILTERITEM=$3
FILTERVALUE=$4
MAXITEMS=$5
DATAITEMLIST=$6

IFS=', ' read -r -a DATAITEMS <<< "$DATAITEMLIST"

DATAITEMQUERY=""
for item in "${DATAITEMS[@]}"
do
  DATAITEMQUERY="$DATAITEMQUERY"",\\(.${item}.S)"
done

# Remove leading ,
DATAITEMQUERY=$(echo "$DATAITEMQUERY" | sed 's/^,//')
# Put in quotes
DATAITEMQUERY=\"${DATAITEMQUERY}\"
#echo $DATAITEMQUERY

DATA=$(aws dynamodb scan --table-name $TABLE --filter-expression "${FILTERITEM} = :itemvalue" --expression-attribute-values "{\":itemvalue\":{\"S\":\"${FILTERVALUE}\"}}" --projection-expression "${DATAITEMLIST}" --profile $PROFILE --limit $MAXITEMS)

ITEMS=$(echo $DATA | jq -r ".Items | .[] | ${DATAITEMQUERY}")

#echo $DATA | jq .
#echo "------"
if [ ! -z "$ITEMS" ]; then
  printf "%s\n" "$ITEMS"
fi

#echo "------"

NEXTTOKEN=$(echo $DATA | jq '.LastEvaluatedKey')
#echo $NEXTTOKEN
#echo "----------------"

while [[ "${NEXTTOKEN}" != "null" ]]
do

ARG="${NEXTTOKEN//\'/$q_mid}"

DATA=$(aws dynamodb scan --table-name $TABLE --filter-expression "$FILTERITEM = :itemvalue" --expression-attribute-values "{\":itemvalue\":{\"S\":\"${FILTERVALUE}\"}}" --exclusive-start-key "${ARG}" --projection-expression "$DATAITEMLIST" --profile $PROFILE --limit $MAXITEMS)

ITEMS=$(echo $DATA | jq -r ".Items | .[] | ${DATAITEMQUERY}")

#echo $DATA | jq .
#echo "------"
if [ ! -z "$ITEMS" ]; then
  printf "%s\n" "$ITEMS"
fi
#echo "------"

NEXTTOKEN=$(echo $DATA | jq '.LastEvaluatedKey')
#echo $NEXTTOKEN
#echo "------------"
sleep 1

done