capture host performance following a docker logs $container_name match

capture-host-performance-docker

#!/bin/bash

## capture-host-performance
## docker logs edition
## Maintainer: Kyle Squizzato - kyle.squizzato@docker.com

## Simple tool to capture performance statistics around a docker logs match.

## Invoke the script and wait for the issue to occur, the script will stop
## on it's own when $match is seen in the $container_name log file.

usage() {
    echo "Usage: $0 [-m <match string>] [-w <wait time in sec>] [-i <interval in sec>] [-C <case number>] [-n <container name>]" 1>&2; exit 0;
}

# Ensure sysstat is installed prior to running so we actually capture
# data when we get to the needed point in time
check_for_stats() {
    if [[ ! -e /usr/bin/iostat && -e /usr/bin/top && -e /usr/bin/vmstat ]]; then
        echo -e '\E[1;37m'"ERROR: The sysstat package does not appear installed, please install it then re-run this script!"
        exit 1
    fi
}

# Ensure the container were going to query against actually exists and can
# be used
check_for_container() {
    if ! docker logs --tail 1 $container_name &> /dev/null; then
        echo -e '\E[1;37m'"ERROR: Unable to query $container_name for logs, is it running?"
        exit 1
    fi
}

while getopts 'm:w:i:C:n:' flag; do
    case "${flag}" in
        # match string
        m) match="${OPTARG}" ;;
        # amount of time in seconds to capture stats following match
        w) wait_="${OPTARG}" ;;
        # interval to capture stats for in seconds
        i) interval="${OPTARG}" ;;
        # case number
        C) casenumber="${OPTARG}" ;;
        # name of container with logs to watch
        n) container_name="${OPTARG}" ;;
        *) usage ;;
    esac
done

if [ -z "${match}" ] || [ -z "${wait_}" ] ||  [ -z "${interval}" ] ||  [ -z "${casenumber}" ] ||  [ -z "${container_name}" ]; then
    usage
fi

# Run checks
check_for_container
check_for_stats

# Start waiting for $match
echo "Waiting for '$match' to show up in docker logs for $container_name"
mkdir perfstats 2&> /dev/null

# Watch for $match in $container_name log then capture some performance
# statistics
docker logs -f $container_name &> /dev/null |
while read line
do
        ret=`echo $line | grep "$match"`
        if [[ -n $ret ]]
        then
                echo -e '\E[1;32m'"Match found, capturing performance statistics for "$wait_" seconds"; tput sgr0
                # Start capturing
                iostat -x $interval > perfstats/iostat.out &
                iostat_pid=$!
                vmstat $interval > perfstats/vmstat.out &
                vmstat_pid=$!
                top -d $interval -b > perfstats/top.out &
                top_pid=$!
                # Sleep
                sleep $wait_
                # Kill the stat captures
                echo "Terminating performance capture..."
                kill $iostat_pid
                kill $vmstat_pid
                kill $top_pid
                break
        fi
done

# Validate whether we actually captured anything in $ret before marking as complete
if [[ $ret == "" ]]
then
	echo -e '\E[1;37m'"ERROR: The capture was interrupted, is $container_name restarting or no longer running?"; tput sgr0
	exit
fi

# tar everything together, we don't clean the resulting directories in
# case the tarball creation doesn't go right we won't lose the data
if [ -e /bin/tar ]; then
        echo "Creating a tarball of logs and captured performance statistics"
        journalctl -u docker --since=today > journalctl_logs.out
        docker logs -t --tail 2000 $container_name &> docker_logs_$container_name.out
        tar czvf $casenumber-$(hostname)-perfstats.tar.gz journalctl_logs.out docker_logs_$container_name.out perfstats/
fi

echo -e "\n "
echo -e '\E[1;31m'"COMPLETE: Please upload perfstats tarball to Docker for analysis."; tput sgr0