transfer-data-s3-to-gs.sh

#!/bin/bash


# Notes

# *** need to export GCP Service account  insdie the script 

# pull data from S3 to local

DIR="/path/to/"
DATE=`date +%Y-%m-%d`


if [ ! -d $DIR ]; then 

  echo "$DIR not found...creating..."
  mkdir -p ${DIR} 
fi 

echo "download files from s3..."
sleep 3

aws s3 cp s3://path/to/ ${DIR} --exclude "*" --include "*${DATE}*" --recursive

if [ $? = 0 ]; then 

   echo "pull files completed succesfully" 

else
   echo "failed to pull files!"     
   exit 1


echo "store unzip files.."
sleep 3

STORE="/path/to/"

if [ ! -d $STORE ]; then 

  echo "$STORE not found...creating..."
  mkdir -p ${STORE} 
fi 

echo "unzip all files..."
sleep 3
cd ${DIR} && *.zip -d ${STORE}


echo "upload files to google storage.."
sleep 3

gsutil cp  "${STORE}/*.csv"  gs://path/to/


echo "cleanup files..."

find $DIR/*.csv.zip  -exec rm {} \;

find $STORE/*.csv  -exec rm {} \;

exit 0