Create a gist now

Instantly share code, notes, and snippets.

@maslade /.gitignore Secret
Last active Dec 21, 2015

What would you like to do?
Fetch and analyze production access logs.
archives
#!/bin/bash
usage() {
echo "Usage: "
echo ""
echo "$0 --help | [--renew] <directory>"
echo ""
echo "Analyzes log files in the specified directory and save the results in the same directory. If results"
echo "results files already exist they will not be recreated. Use --renew to regenerate results files."
}
make_or_skip() {
if [ -e "$1" ]; then
if (($OPT_CLEANUP)); then
echo " ...regenerating $1"
return 1
else
echo " ...skipping pre-existing $1"
return 0
fi
else
echo " ...generating $1"
return 1
fi
}
OPT_CLEANUP=0
OPT_LOG=
while (("$#")); do
case "$1" in
-r|--renew)
OPT_CLEANUP=1
;;
-h|--help)
usage
exit 0
;;
*)
if [ -z $OPT_LOG ]; then
OPT_LOG="$1"
else
echo "Unrecognized option: $1"
usage
exit 1
fi
;;
esac
shift
done
DEST=${OPT_LOG:-$(pwd)}
APPS=$(ls ${DEST}/*-app-*.gz 2>/dev/null)
ASSETS=$(ls ${DEST}/*-assets-*.gz 2>/dev/null)
if [ ! -w "$DEST" ]; then
echo "Unable to write results to $DEST."
exit 3
fi
if [ -z "$APPS" ]; then
echo "No app log files found in $DEST."
exit 3
fi
if [ -z "$ASSETS" ]; then
echo "No asset log files found in $DEST."
exit 3
fi
OUT_APP_REQ="$DEST/apps-access.log.gz"
OUT_ASSETS_REQ="$DEST/assets-access.log.gz"
OUT_REQ="$DEST/requests.txt"
TALLY_REQ="$DEST/wp-admin-requests-tally.txt"
TALLY_GLOB_REQ="$DEST/wp-admin-global-requests-tally.txt"
TALLY_ADMIN_ACTION="$DEST/wp-admin-ajax-action-tally.txt"
TALLY_GLOB_ADMIN_ACTION="$DEST/wp-admin-global-ajax-action-tally.txt"
# Combine server logs
make_or_skip "$OUT_APP_REQ" || \
gunzip -c $APPS \
| gzip -9 - \
> $OUT_APP_REQ
make_or_skip "$OUT_ASSETS_REQ" || \
gunzip -c $ASSETS \
| gzip -9 - \
> $OUT_ASSETS_REQ
# App requests
make_or_skip "$OUT_REQ" || \
gunzip -c $OUT_APP_REQ \
| cut -d ' ' -f 7 \
> $OUT_REQ
# Per-site request tally
make_or_skip "$TALLY_REQ" || \
grep wp-admin $OUT_REQ \
| sed 's/"//' \
| sed 's/\?.*$//' \
| sort \
| uniq --count \
| sort -n -k 1 \
> $TALLY_REQ
# Global request tally
make_or_skip "$TALLY_GLOB_REQ" || \
grep wp-admin $OUT_REQ \
| sed 's/"//' \
| sed 's/\/[^\/]*\//\/<site>\//' \
| sed 's/\?.*$//' \
| sort \
| uniq --count \
| sort -n -k 1 \
> $TALLY_GLOB_REQ
# Per-site wp-admin action tally
make_or_skip "$TALLY_ADMIN_ACTION" || \
grep 'admin-ajax.php' $OUT_REQ \
| grep 'action=' \
| sort \
| uniq -c \
| sort -n -k1 \
> $TALLY_ADMIN_ACTION
# Global wp-admin action tally
make_or_skip "$TALLY_GLOB_ADMIN_ACTION" || \
grep 'admin-ajax.php' $OUT_REQ \
| grep 'action=' \
| sed 's/\/[^\/]*\//<site>\//' \
| sort \
| uniq -c \
| sort -n -k1 \
> $TALLY_GLOB_ADMIN_ACTION
# Summary
c=5
num_asset_reqs=$(gunzip -c $OUT_ASSETS_REQ | wc -l)
num_app_reqs=$(cat $OUT_REQ | wc -l)
echo
echo "Completed. Here is a quick overview:"
echo "================================================"
printf "%-40s%d\n" "Total requests:" $((${num_asset_reqs} + ${num_app_reqs}))
printf "%-40s%d\n" " Asset requests:" $num_asset_reqs
printf "%-40s%d\n" " App requests:" $num_app_reqs
printf "%-40s%d\n" " WP Admin requests:" $(cat $OUT_REQ | grep /wp-admin/ | wc -l)
printf "%-40s%d\n" " admin-ajax requests:" $(grep admin-ajax.php $OUT_REQ -c)
echo
echo "Top global requests (ignoring admin-ajax):"
cat $TALLY_GLOB_REQ | grep -v admin-ajax.php | tail -n $c
echo
echo "Top per-site requests (ignoring admin-ajax):"
cat $TALLY_REQ | grep -v admin-ajax.php | tail -n $c
echo
echo "Top global admin-ajax actions:"
tail -n $c $TALLY_GLOB_ADMIN_ACTION
echo
echo "Top per-site admin-ajax actions:"
tail -n $c $TALLY_ADMIN_ACTION
echo
#!/bin/bash
#
# Retrieve log files from yesterday from production servers. Includes app and assets.
# Results storedin archives/<DATESTAMP> as gzipped files.
YESTERDAY=$(date -d "1 day ago" +"%m-%d-%Y")
YESTERDAYSTRING=$(date -d "1 day ago" +"%Y%m%d")
HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DEST="${HERE}/archives/${YESTERDAY}"
if [ ! -d $DEST ]; then
mkdir -p $DEST \
|| exit 1
fi
SERVERS=(ist-wp-app-prod01 ist-wp-app-prod02 ist-wp-app-prod03)
for server in ${SERVERS[@]}; do
TARGET="$DEST/$server-app-access_log.gz"
FILE="/var/log/apache/wp-app-prod/access_log*${YESTERDAYSTRING}*.gz"
echo "Copy $server:$FILE to $TARGET"
scp "$server:$FILE" "$TARGET"
TARGET="$DEST/$server-assets-access_log.gz"
FILE="/var/log/apache/wp-assets-prod/access_log*${YESTERDAYSTRING}*.gz"
echo "Copy $server:$FILE to $TARGET"
scp "$server:$FILE" "$TARGET"
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment