Simple tool to capture performance statistics around a Docker daemon log match.

capture-host-perf.sh

#!/bin/bash

## capture-host-performance
## Maintainer: Kyle Squizzato - kyle.squizzato@docker.com

## Simple tool to capture performance statistics around a daemon log match.
## Only supports distros using journalctl at this time.

## Fill in each of the variables in the SETUP section then invoke the script
## and wait for the issue to occur, the script will stop on it's own when
## the $match is seen in the $log file.

## -------- SETUP ---------

# Message to match from daemon log
match="sync duration of"

# Amount of time in seconds to capture performance statistics for following
# match
wait_=300

# Interval to capture performance statistics for, default 5s
interval=5

# Case number
casenumber="00023066"

## -------- END SETUP ---------

# Ensure sysstat is installed prior to running so we actually capture
# data when we get to the needed point in time
check_for_stats() {
    if [[ ! -e /usr/bin/iostat && -e /usr/bin/top && -e /usr/bin/vmstat ]]; then
        echo "ERROR: The sysstat package does not appear installed, please install it then re-run this script!"
        exit
    fi
}

check_for_stats
echo "Waiting for '$match' to show up in Docker daemon logs"
mkdir perfstats 2&> /dev/null

# Watch for $match in $log then capture some performance statistics
journalctl -fu docker -n 1 |
while read line
do
        ret=`echo $line | grep "$match"`
        if [[ -n $ret ]]
        then
                echo -e '\E[1;32m'"Match found, capturing performance statistics for "$wait_" seconds"; tput sgr0
                # Start capturing
                iostat -x $interval > perfstats/iostat.out &
                iostat_pid=$!
                vmstat $interval > perfstats/vmstat.out &
                vmstat_pid=$!
                top -d $interval -b > perfstats/top.out &
                top_pid=$!
                # Sleep
                sleep $wait_
                # Kill the stat captures
                echo "Terminating performance capture..."
                kill $iostat_pid
                kill $vmstat_pid
                kill $top_pid
                break
        fi
done

# tar everything together, we don't clean the resulting directories in
# case the tarball creation doesn't go right we won't lose the data
if [ -e /bin/tar ]; then
        echo "Creating a tarball of daemon logs and captured performance statistics"
        journalctl -u docker --since=today > journalctl_logs.out
        tar czvf $casenumber-$(hostname)-perfstats.tar.gz journalctl_logs.out perfstats/
fi

echo -e "\n "
echo -e '\E[1;31m'"COMPLETE: Please upload perfstats tarball to Docker for analysis."; tput sgr0