Wrapper for redshift_unload_copy.py to unload/copy multiple tables in a loop

redshift_unload_wrapper.sh

#!/bin/bash

# Wylie Hobbs - 2018
# Copy multiple tables at once with redshift unload-copy utility
# Usage: ./redshift_unload_wrapper.sh
# Requires a clone of amazon-redshift-utils and the packages: jq, awscli
# Create redshift_unload_wrapper.sh, tables.txt and config.json in src/UnloadCopyUtility and run this script

#tables.txt is newline delimited list of tables in SCHEMA.TABLE notation
TABLES=$(cat tables.txt)
CONFIG="config.json"
S3_BUCKET="some-bucket"
REGION="us-west-2"

for st in ${TABLES[@]}; do
  SCHEMA=$(echo $st | cut -f1 -d.)
  TABLE=$(echo $st | cut -f2 -d.)
  TMP_CONFIG="${TABLE}_config.json"

  if [ "$TMP_CONFIG" != ".json" ]; then
    jq ".unloadSource.schemaName=\"$SCHEMA\" | .copyTarget.schemaName=\"$SCHEMA\" | .unloadSource.tableName=\"$TABLE\" | .copyTarget.tableName=\"$TABLE\"" $CONFIG > $TMP_CONFIG

    echo "Uploading config $TMP_CONFIG to S3"
    aws s3 cp $TMP_CONFIG s3://$S3_BUCKET/$TMP_CONFIG

    echo "Running redshift-unload-copy for $st"
    python redshift_unload_copy.py --s3-config-file s3://$S3_BUCKET/$TMP_CONFIG --destination-table-auto-create True --region $REGION

    if [ $? -ne 0 ]; then
      echo "Something went wrong. Exiting"
      exit 1
    fi
  else
    echo "Temp config is empty! Exiting"
    exit 1
  fi
done