https://github.com/glennklockwood/bioinformatics-profile - Adapted for SLURM

peoples_profiler.sh

#!/bin/bash
# Based heavily on: https://github.com/glennklockwood/bioinformatics-profile/blob/master/generic-profiler.sh
# Environment variables for configuring behavior:
#   NO_PROFILE=1 to skip profiling altogether
#   NO_DF=1 to skip profiling file system capacities
#   NO_FILEHANDLES=1 to skip profiling file handle counts
#     
NO_DF=1
NO_FILEHANDLES=1

PROFILE_INTERVAL=5
BINARY="w2rap-contigger"
THREADS=$SLURM_CPUS_PER_TASK
APWRAP=""
APP_PGREP="$(basename $BINARY)"

clean_profile_dir() {
    #
    #  Ensure we don't carry over the results from a previous profiling run
    #
    PROFILE_OUTPUT_DIR=$1
    if [ -d $PROFILE_OUTPUT_DIR ]; then
        echo "$(date) - Need to kill $PROFILE_OUTPUT_DIR"
        if [ -d ${PROFILE_OUTPUT_DIR}.old ]; then
            rm -rf ${PROFILE_OUTPUT_DIR}.old
        fi
        mv -v $PROFILE_OUTPUT_DIR ${PROFILE_OUTPUT_DIR}.old
    fi
    mkdir -p $PROFILE_OUTPUT_DIR
}

PROFILE_OUTPUT_DIR="~/$SLURM_JOBID/profile"

drop_begin() {
    if [ -z "$1" ]; then
        echo "PROF_BEGIN $1"
    else
        echo "PROF_BEGIN $(date +%s)"
    fi
}

startmon() { 
    echo "Starting IO profile..."


    for profile_output in prof_df.txt prof_ps.txt prof_filehandles.txt prof_vmstat.txt prof_meminfo.txt prof_gstack.txt
    do
        if [ -e $PROFILE_OUTPUT_DIR/$profile_output ]; then
            rm $PROFILE_OUTPUT_DIR/$profile_output
        fi
    done
    drop_begin > $PROFILE_OUTPUT_DIR/prof_iostat.txt
    iostat -dkt 30 $SCRATCH_DEV >> $PROFILE_OUTPUT_DIR/prof_iostat.txt &

    if [ -e $PROFILE_OUTPUT_DIR/prof_df.txt ]; then
    rm $PROFILE_OUTPUT_DIR/prof_df.txt
    fi
    if [ -e $PROFILE_OUTPUT_DIR/prof_ps.txt ]; then
    rm $PROFILE_OUTPUT_DIR/prof_ps.txt
    fi
    while [ 1 ]
    do 
        timestamp=$(date +%s)
        # save record of ssd capacity
        drop_begin $timestamp >> $PROFILE_OUTPUT_DIR/prof_df.txt
        df -k >> $PROFILE_OUTPUT_DIR/prof_df.txt

        if [ -z "$NO_FILEHANDLES" ]; then
            drop_begin $timestamp >> $PROFILE_OUTPUT_DIR/prof_filehandles.txt
            cat /proc/sys/fs/file-nr >> $PROFILE_OUTPUT_DIR/prof_filehandles.txt
        fi
        # save record of virtual memory state
        drop_begin $timestamp >> $PROFILE_OUTPUT_DIR/prof_vmstat.txt
        cat /proc/vmstat >> $PROFILE_OUTPUT_DIR/prof_vmstat.txt

        # save record of memory
        drop_begin $timestamp >> $PROFILE_OUTPUT_DIR/prof_meminfo.txt
        cat /proc/meminfo >> $PROFILE_OUTPUT_DIR/prof_meminfo.txt

        # save record of running processes
        drop_begin $timestamp >> $PROFILE_OUTPUT_DIR/prof_ps.txt
        ps -U $USER -o pid,ppid,lwp,nlwp,etime,pcpu,pmem,rss,vsz,cmd -www >> $PROFILE_OUTPUT_DIR/prof_ps.txt

        # only probe the process stack if we are doing coarse-grained profiling
        if [ ${PROFILE_INTERVAL} -ge 5 ]; then
            my_pid=$(pgrep $APP_PGREP | head -n1)
            if [ ! -z "$my_pid" ]; then
                drop_begin $timestamp >> $PROFILE_OUTPUT_DIR/prof_gstack.txt
                gstack $my_pid 2>&1 >> $PROFILE_OUTPUT_DIR/prof_gstack.txt
            fi
        fi

        sleep ${PROFILE_INTERVAL}s


    done
}
###########################################################################################
#
#  EVERYTHING ABOVE HERE BELONGS ON THE TASK-PROLOGUE
#
###########################################################################################

clean_profile_dir $PROFILE_OUTPUT_DIR


if [ ! $NO_PROFILE ]; then
    startmon &
    monpid=$!
    sleep 5
fi

echo "$(date) - Running command (check stderr for invocation)"
set -x

# Your executable gets called here, set the parameters, and exports required here...
export OMP_NUM_THREADS=$THREADS
$APWRAP $BINARY \
                -t $THREADS \
                --crazy-param 123 \
                --sane-param 2

set +x
echo "$(date) - Finished running command"



############################################################################################
#
# EVERYTHING BELOW HERE CAN BE MOVED TO THE TASK-EPILOGUE
#
############################################################################################

# This can be moved to the epilogue
if [ ! $NO_PROFILE ]; then
    ### Let one last ps/df fire before shutting everything down
    sleep 30
    kill $monpid
fi


echo "$(date) - Begin moving output data off of local disk"
mv -v $SCRATCH_DIR/$OUTPUT_FILE $OUTPUT_DIR/
mv -v $SCRATCH_DIR/${OUTPUT_FILE%out}err $OUTPUT_DIR/
echo "$(date) - Finished moving output data off of local disk"

echo "$(date) - Removing $SCRATCH_DIR"
rm -rf $SCRATCH_DIR
echo "$(date) - Done cleaning up $SCRATCH_DIR"

# END