Fetch and analyze production access logs.

.gitignore

archives

analyze.sh

#!/bin/bash

usage() {
        echo "Usage: "
        echo ""
        echo "$0 --help | [--renew] <directory>"
        echo ""
        echo "Analyzes log files in the specified directory and save the results in the same directory.  If results"
        echo "results files already exist they will not be recreated.  Use --renew to regenerate results files."
}

make_or_skip() {
        if [ -e "$1" ]; then
                if (($OPT_CLEANUP)); then
                        echo " ...regenerating $1"
                        return 1
                else
                        echo " ...skipping pre-existing $1"
                        return 0
                fi
        else
                echo " ...generating $1"
                return 1
        fi
}

OPT_CLEANUP=0
OPT_LOG=

while (("$#")); do
        case "$1" in

        -r|--renew)
                OPT_CLEANUP=1
                ;;

        -h|--help)
                usage
                exit 0
                ;;

        *)
                if [ -z $OPT_LOG ]; then
                        OPT_LOG="$1"
                else
                        echo "Unrecognized option: $1"
                        usage
                        exit 1
                fi
                ;;
        esac

        shift
done

DEST=${OPT_LOG:-$(pwd)}
APPS=$(ls ${DEST}/*-app-*.gz 2>/dev/null)
ASSETS=$(ls ${DEST}/*-assets-*.gz 2>/dev/null)
if [ ! -w "$DEST" ]; then
        echo "Unable to write results to $DEST."
        exit 3
fi

if [ -z "$APPS" ]; then
        echo "No app log files found in $DEST."
        exit 3
fi

if [ -z "$ASSETS" ]; then
        echo "No asset log files found in $DEST."
        exit 3
fi

OUT_APP_REQ="$DEST/apps-access.log.gz"
OUT_ASSETS_REQ="$DEST/assets-access.log.gz"
OUT_REQ="$DEST/requests.txt"
TALLY_REQ="$DEST/wp-admin-requests-tally.txt"
TALLY_GLOB_REQ="$DEST/wp-admin-global-requests-tally.txt"
TALLY_ADMIN_ACTION="$DEST/wp-admin-ajax-action-tally.txt"
TALLY_GLOB_ADMIN_ACTION="$DEST/wp-admin-global-ajax-action-tally.txt"


# Combine server logs
make_or_skip "$OUT_APP_REQ" || \
        gunzip -c $APPS \
         | gzip -9 - \
         > $OUT_APP_REQ
make_or_skip "$OUT_ASSETS_REQ" || \
        gunzip -c $ASSETS \
         | gzip -9 - \
         > $OUT_ASSETS_REQ

# App requests
make_or_skip "$OUT_REQ" || \
        gunzip -c $OUT_APP_REQ \
         | cut -d ' ' -f 7 \
         > $OUT_REQ

# Per-site request tally
make_or_skip "$TALLY_REQ" || \
        grep wp-admin $OUT_REQ \
         | sed 's/"//' \
         | sed 's/\?.*$//' \
         | sort \
         | uniq --count \
         | sort -n -k 1 \
         > $TALLY_REQ

# Global request tally
make_or_skip "$TALLY_GLOB_REQ" || \
        grep wp-admin $OUT_REQ \
         | sed 's/"//' \
         | sed 's/\/[^\/]*\//\/<site>\//' \
         | sed 's/\?.*$//' \
         | sort \
         | uniq --count \
         | sort -n -k 1 \
         > $TALLY_GLOB_REQ

# Per-site wp-admin action tally
make_or_skip "$TALLY_ADMIN_ACTION" || \
        grep 'admin-ajax.php' $OUT_REQ \
         | grep 'action=' \
         | sort \
         | uniq -c \
         | sort -n -k1 \
        > $TALLY_ADMIN_ACTION

# Global wp-admin action tally
make_or_skip "$TALLY_GLOB_ADMIN_ACTION" || \
        grep 'admin-ajax.php' $OUT_REQ \
         | grep 'action=' \
         | sed 's/\/[^\/]*\//<site>\//' \
         | sort \
         | uniq -c \
         | sort -n -k1 \
        > $TALLY_GLOB_ADMIN_ACTION

# Summary
c=5
num_asset_reqs=$(gunzip -c $OUT_ASSETS_REQ | wc -l)
num_app_reqs=$(cat $OUT_REQ | wc -l)
echo
echo "Completed.  Here is a quick overview:"
echo "================================================"
printf "%-40s%d\n" "Total requests:" $((${num_asset_reqs} + ${num_app_reqs}))
printf "%-40s%d\n" "    Asset requests:" $num_asset_reqs
printf "%-40s%d\n" "    App requests:" $num_app_reqs
printf "%-40s%d\n" "        WP Admin requests:" $(cat $OUT_REQ | grep /wp-admin/ | wc -l)
printf "%-40s%d\n" "            admin-ajax requests:" $(grep admin-ajax.php $OUT_REQ -c)
echo
echo "Top global requests (ignoring admin-ajax):"
cat $TALLY_GLOB_REQ | grep -v admin-ajax.php | tail -n $c
echo
echo "Top per-site requests (ignoring admin-ajax):"
cat $TALLY_REQ | grep -v admin-ajax.php | tail -n $c
echo
echo "Top global admin-ajax actions:"
tail -n $c $TALLY_GLOB_ADMIN_ACTION
echo
echo "Top per-site admin-ajax actions:"
tail -n $c $TALLY_ADMIN_ACTION
echo

fetch.sh

#!/bin/bash
#
# Retrieve log files from yesterday from production servers.  Includes app and assets.
# Results storedin archives/<DATESTAMP> as gzipped files.

YESTERDAY=$(date -d "1 day ago" +"%m-%d-%Y")
YESTERDAYSTRING=$(date -d "1 day ago" +"%Y%m%d")
HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
DEST="${HERE}/archives/${YESTERDAY}"

if [ ! -d $DEST ]; then
        mkdir -p $DEST \
         || exit 1
fi

SERVERS=(ist-wp-app-prod01 ist-wp-app-prod02 ist-wp-app-prod03)

for server in ${SERVERS[@]}; do
        TARGET="$DEST/$server-app-access_log.gz"
        FILE="/var/log/apache/wp-app-prod/access_log*${YESTERDAYSTRING}*.gz"
        echo "Copy $server:$FILE to $TARGET"
        scp "$server:$FILE" "$TARGET"

        TARGET="$DEST/$server-assets-access_log.gz"
        FILE="/var/log/apache/wp-assets-prod/access_log*${YESTERDAYSTRING}*.gz"
        echo "Copy $server:$FILE to $TARGET"
        scp "$server:$FILE" "$TARGET"
done