Download bucket files and save them in folders

get_files.sh

len=${#buckets[@]}
i=0

while [ $len -gt $i ]
do
  bucket=${buckets[$i]}
  url=${urls[$i]}
  echo $bucket
  mkdir -p $bucket && cd $bucket
  filenames=$(aws s3 ls s3://${buckets[$i]} | awk '{ print $4 }')

  j=0
  for fls in $filenames
  do
    (`wget -qc ${url}${fls} -O ${fls}`) &
    j=$(( j+1 ))
    if (( $j%10 == 0 ))
    then
      wait ${!}
    fi
  done
  echo "WAITING....."
  wait ${!}

  cd ..
  i=$(( i+1 ))
done