get_perf_stats.sh

#!/bin/bash
# Description: wait until system load reaches threshold then start
#              scripts and run for period then kill.
#
LIMIT=90
RUNTIME=300
SESSION_ID=`uuidgen`
declare -A PIDS=()

log () { echo -e "[`date '+%F %X'`] $@"; }
load () { cat /proc/loadavg| awk '{print $1}'; }
loop () { cmd="$1"; outfile=$2; period=${3:-1}; while true; do $cmd >> $outfile; sleep $period; done; }

if [[ `whoami` != root ]]; then
    echo "ERROR: must be run as root"
    exit 1
fi

DEPS=(
  lm-sensors
  linux-tools-common
  cpufrequtils
  sysstat
)
log "Ensuring deps installed: ${DEPS[@]}"
for dep in ${DEPS[@]}; do
    dpkg -s $dep &>/dev/null || apt install $dep
done

log "Staring session ${SESSION_ID}"
while true; do
    _load=$(load)
    log "Waiting for system load to reach $LIMIT before staring perf record (current=$(load))."
    if ((${_load%.*} >= LIMIT)); then
        log "Starting scripts record now that system load has reached $_load"
        log "Running perf record for ${RUNTIME}s before stopping (system load=$(load))"
        perf record -F 99 -a -g -d --sample-cpu -- sleep $RUNTIME &
        PERF_PID=$!

        log "Starting extra monitoring utlities (system load=$(load))"
        mpstat -P ALL 1 > mpstat.${SESSION_ID}.out &
        PIDS[mpstat]=$!

        cpufreq-aperf -i 1 &> cpufreq-aperf.${SESSION_ID}.out &
        PIDS[cpufreq-aperf]=$!

        pidstat 1 > pidstat.${SESSION_ID}.out &
        PIDS[pidstat]=$!

        loop sensors sensors.${SESSION_ID}.out &
        PIDS[sensors]=$!

        loop cpufreq-info cpufreq-info.${SESSION_ID}.out &
        PIDS[cpufreq-info]=$!

        loop "cpupower monitor" cpupower-monitor.${SESSION_ID}.out &
        PIDS[cpupower-monitor]=$!

        wait $PERF_PID
        # NOTE: assumes all processes will be killed by SIGTERM
        kill ${PIDS[@]}
        for pname in ${!PIDS[@]}; do
            log "checking $pname killed (pid=${PIDS[$pname]})"
            if `ps -p ${PIDS[$pname]}| grep -q $pname`; then
                log "ERROR: process $pname was not successfully killed - requires manual kill"
            fi
        done
        mv perf.data perf_${SESSION_ID}.data
        break
    fi
    sleep 1
done
log "Done."
log "Results:\n`ls *${SESSION_ID}*`"