Skip to content

Instantly share code, notes, and snippets.

@JanTvrdik
Last active Jun 23, 2022
Embed
What would you like to do?
Papertrail S3 log downloader

Papertrail S3 log downloader

  1. Get Your Papertrail token from https://papertrailapp.com/account/profile
  2. Profit!
# download logs from last 30 days
PAPERTRAIL_TOKEN=9X4cddgwe53fAbbsYh4 papertrail-download-daily.sh 30

# download logs from last 30 days & filter each through ./filter.sh
PAPERTRAIL_TOKEN=9X4cddgwe53fAbbsYh4 papertrail-download-daily.sh 30 ./filter.sh

# download logs from last 168 hours
PAPERTRAIL_TOKEN=9X4cddgwe53fAbbsYh4 papertrail-download-hourly.sh 168
#!/usr/bin/env bash
set -o errexit -o pipefail -o nounset
# arguments
DAYS_COUNT="$1"
FILTER="${2:-cat}"
seq 1 "$DAYS_COUNT" | tac | xargs -I {} date -u --date='{} day ago' +%Y-%m-%d | \
xargs --max-procs 4 --max-args 1 -I {} ./papertrail-download-single.sh {} "$FILTER"
wait
#!/usr/bin/env bash
set -o errexit -o pipefail -o nounset
# arguments
HOURS_COUNT="$1"
FILTER="${2:-cat}"
seq 1 "$HOURS_COUNT" | tac | xargs -I {} date -u --date='{} hours ago' +%Y-%m-%d-%H | \
xargs --max-procs 8 --max-args 1 -I {} ./papertrail-download-single.sh {} "$FILTER"
wait
#!/usr/bin/env bash
set -o pipefail -o nounset
# arguments
FILE_NAME="$1"
FILTER="${2:-cat}"
# local variables
OUTPUT_PATH="$FILE_NAME.tsv"
OUTPUT_PATH_TMP=="$OUTPUT_PATH.tmp"
OUTPUT_PATH_UNFILTERED="$OUTPUT_PATH.unfiltered"
OUTPUT_PATH_GZ="$OUTPUT_PATH_UNFILTERED.gz"
OUTPUT_PATH_GZ_TMP="$OUTPUT_PATH_GZ.tmp"
if [ -f "$OUTPUT_PATH" ]; then
echo -n "." # skipped
elif [ -f "$OUTPUT_PATH_GZ" ]; then
gunzip "$OUTPUT_PATH_GZ" && \
"$FILTER" < "$OUTPUT_PATH_UNFILTERED" > "$OUTPUT_PATH_TMP" && \
mv "$OUTPUT_PATH_TMP" "$OUTPUT_PATH" && \
rm "$OUTPUT_PATH_UNFILTERED" && \
echo -n "E" # extracted
elif [ -f "$OUTPUT_PATH_UNFILTERED" ]; then
"$FILTER" < "$OUTPUT_PATH_UNFILTERED" > "$OUTPUT_PATH_TMP" && \
mv "$OUTPUT_PATH_TMP" "$OUTPUT_PATH" && \
rm "$OUTPUT_PATH_UNFILTERED" && \
echo -n "T" # transformed
else
rm -f "OUTPUT_PATH_TMP" "$OUTPUT_PATH_GZ_TMP" && \
curl -f --no-include --fail --retry 3 --silent -o "$OUTPUT_PATH_GZ_TMP" -L -H "X-Papertrail-Token: $PAPERTRAIL_TOKEN" "https://papertrailapp.com/api/v1/archives/$FILE_NAME/download"
DOWNLOAD_RESULT=$?
if [ $DOWNLOAD_RESULT -eq 0 ]; then
mv "$OUTPUT_PATH_GZ_TMP" "$OUTPUT_PATH_GZ" && \
gunzip "$OUTPUT_PATH_GZ" && \
"$FILTER" < "$OUTPUT_PATH_UNFILTERED" > "$OUTPUT_PATH_TMP" && \
mv "$OUTPUT_PATH_TMP" "$OUTPUT_PATH" && \
rm "$OUTPUT_PATH_UNFILTERED" && \
echo -n ":" # downloaded
elif [ $DOWNLOAD_RESULT -eq 22 ]; then
touch "$OUTPUT_PATH" && \
echo -n "N" # not found
else
echo -n "F" # failed
fi
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment