Download, gunzip, and concatenate a directory of (CSV) files from S3

unzipS3.sh

#! /bin/bash

REMOTE_DIR=$1
LOCAL_FILE=$2

if [ -e $LOCAL_FILE ]; then
  echo "File exists: $LOCAL_FILE"
  exit -1
fi

for f in $(aws s3 ls "$REMOTE_DIR/" | sed -n 's/.* \(part.*gz\)/\1/p'); do
  path="$REMOTE_DIR/$f"
  tmp=$(mktemp /tmp/XXXXXXXX)

  aws s3 cp $path $tmp.gz
  ls -l $tmp.gz

  gunzip -f $tmp.gz
  ls -l $tmp

  cat $tmp >> $LOCAL_FILE

  rm $tmp
done

ls -l $LOCAL_FILE
wc -l $LOCAL_FILE