Download Papertrail archives, upload to S3

async_papertrail_manual_archive.sh

#!bin/bash
# Downloads 10 days of logs at a time, asynchronously. 
# Env vars:
# MAX_DAYS: Number of days of logs to download
# PAPERTRAIL_API_KEY: Self explanitory, get from PT "Profile" page
# BUCKET_NAME: S3 Bucket to upload to
# LOG_PATH: Path inside S3 Bucket to save logs to

for i in $(seq 0 ${MAX_DAYS})
do
  date -u -v-${i}d +%Y-%m-%d | \
  xargs -I {} curl --progress-bar -f --no-include --create-dirs -o dt={}/{}.tsv.gz \
  -L -H "X-Papertrail-Token: ${PAPERTRAIL_API_KEY}" https://papertrailapp.com/api/v1/archives/{}/download &
  if (( $i % 10 == 0 )); then wait; fi
done
wait

s3cmd put --recursive . s3://${BUCKET_NAME}/${LOG_PATH}/