Papertrail S3 log downloader

README.md

Papertrail S3 log downloader

1. Get Your Papertrail token from https://papertrailapp.com/account/profile
2. Profit!

# download logs from last 30 days
PAPERTRAIL_TOKEN=9X4cddgwe53fAbbsYh4 papertrail-download-daily.sh 30

# download logs from last 30 days & filter each through ./filter.sh
PAPERTRAIL_TOKEN=9X4cddgwe53fAbbsYh4 papertrail-download-daily.sh 30 ./filter.sh

# download logs from last 168 hours
PAPERTRAIL_TOKEN=9X4cddgwe53fAbbsYh4 papertrail-download-hourly.sh 168

papertrail-download-daily.sh

#!/usr/bin/env bash
set -o errexit -o pipefail -o nounset

# arguments
DAYS_COUNT="$1"
FILTER="${2:-cat}"


seq 1 "$DAYS_COUNT" | tac | xargs -I {} date -u --date='{} day ago' +%Y-%m-%d | \
    xargs --max-procs 4 --max-args 1 -I {} ./papertrail-download-single.sh {} "$FILTER"
wait

papertrail-download-hourly.sh

#!/usr/bin/env bash
set -o errexit -o pipefail -o nounset

# arguments
HOURS_COUNT="$1"
FILTER="${2:-cat}"


seq 1 "$HOURS_COUNT" | tac | xargs -I {} date -u --date='{} hours ago' +%Y-%m-%d-%H | \
    xargs --max-procs 8 --max-args 1 -I {} ./papertrail-download-single.sh {} "$FILTER"
wait

papertrail-download-single.sh

#!/usr/bin/env bash
set -o pipefail -o nounset

# arguments
FILE_NAME="$1"
FILTER="${2:-cat}"

# local variables
OUTPUT_PATH="$FILE_NAME.tsv"
OUTPUT_PATH_TMP=="$OUTPUT_PATH.tmp"
OUTPUT_PATH_UNFILTERED="$OUTPUT_PATH.unfiltered"
OUTPUT_PATH_GZ="$OUTPUT_PATH_UNFILTERED.gz"
OUTPUT_PATH_GZ_TMP="$OUTPUT_PATH_GZ.tmp"


if [ -f "$OUTPUT_PATH" ]; then
	echo -n "." # skipped

elif [ -f "$OUTPUT_PATH_GZ" ]; then
	gunzip "$OUTPUT_PATH_GZ" && \
	"$FILTER" < "$OUTPUT_PATH_UNFILTERED" > "$OUTPUT_PATH_TMP" && \
	mv "$OUTPUT_PATH_TMP" "$OUTPUT_PATH" && \
	rm "$OUTPUT_PATH_UNFILTERED" && \
	echo -n "E" # extracted

elif [ -f "$OUTPUT_PATH_UNFILTERED" ]; then
	"$FILTER" < "$OUTPUT_PATH_UNFILTERED" > "$OUTPUT_PATH_TMP" && \
	mv "$OUTPUT_PATH_TMP" "$OUTPUT_PATH" && \
	rm "$OUTPUT_PATH_UNFILTERED" && \
	echo -n "T" # transformed

else
	rm -f "OUTPUT_PATH_TMP" "$OUTPUT_PATH_GZ_TMP" && \
	curl -f --no-include --fail --retry 3 --silent -o "$OUTPUT_PATH_GZ_TMP" -L -H "X-Papertrail-Token: $PAPERTRAIL_TOKEN" "https://papertrailapp.com/api/v1/archives/$FILE_NAME/download"
	DOWNLOAD_RESULT=$?

	if [ $DOWNLOAD_RESULT -eq 0 ]; then
		mv "$OUTPUT_PATH_GZ_TMP" "$OUTPUT_PATH_GZ" && \
		gunzip "$OUTPUT_PATH_GZ" && \
		"$FILTER" < "$OUTPUT_PATH_UNFILTERED" > "$OUTPUT_PATH_TMP" && \
		mv "$OUTPUT_PATH_TMP" "$OUTPUT_PATH" && \
		rm "$OUTPUT_PATH_UNFILTERED" && \
		echo -n ":" # downloaded

	elif [ $DOWNLOAD_RESULT -eq 22 ]; then
		touch "$OUTPUT_PATH" && \
		echo -n "N" # not found

	else
		echo -n "F" # failed
	fi
fi