Dumping Isilon (OneFS) stats in a text format for later analysis.

isilon_stats.sh

#!/bin/zsh
#The intent of this script is to be called from a cron job every 5 or 10 minutes on an Isilon node.
#Dumps a bunch of text based stats to a rotating log file every 24 hours. This is useful for troubleshooting purposes.

#Let's first look and see if we have a stuck execution of the script. If the lock is older than 10 minutes we'll kill the PID and cleanup the lock.
find /tmp/nfs-stat.lck -mmin +20 -exec kill -9 `cat /tmp/nfs-stat.lck` {} \;
find /tmp/nfs-stat.lck -mmin +20 -exec rm -rf /tmp/nfs-stat.lck {} \;

lockfile=/tmp/nfs-stat.lck

if ( set -o noclobber; echo "$$" > "$lockfile") 2> /dev/null; then

        trap 'rm -f "$lockfile"; exit $?' INT TERM EXIT

        # do stuff here
        echo TOTAL for $(date "+%d-%m-%Y--%H:%M:%S") > /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics system --degraded --oprates" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics system --degraded --oprates >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics pstat --degraded" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics pstat --degraded  >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics client --degraded --orderby=ops | head -n 32" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics client --degraded --orderby=ops | head -n 32 >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics client --degraded --totalby=username --orderby=ops | head -n 32" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics client --totalby=username --orderby=ops | head -n 32 >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics drive --long --nodes=all --degraded --orderby=TimeInQ | head -22" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics drive --long --nodes=all --degraded --orderby=TimeInQ | head -22 >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics heat --degraded -nall --orderby ops --totalby path --limit 50" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics heat --degraded -nall --orderby ops --totalby path --limit 50  >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics heat --degraded -nall --orderby=path --events=deadlocked" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics heat --degraded -nall --orderby=path --events=deadlocked >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics client --degraded --totalby=class" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics client --degraded --totalby=class >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics heat --degraded -nall --totalby event" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics heat --degraded -nall --totalby event >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics client --degraded -nall --orderby timeavg | head -52" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics client --degraded -nall --orderby timeavg | head -52 >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo "isi statistics protocol --degraded --protocols nfs3 --totalby op" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics protocol --degraded --protocols nfs3 --totalby op >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics protocol --degraded --protocols=nfs3 --long -i 5 -r 1 --nodes all --orderby numops --classes=namespace_write | head -54" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics protocol --degraded --protocols=nfs3 --long -i 5 -r 1 --nodes all --orderby numops --classes=namespace_write | head -54 >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi statistics protocol --degraded --protocols=nfs3 --long -i 5 -r 2 --nodes all --orderby numops --totalby op" >> /ifs/admin/scripts/stat/stats/current.log
        isi statistics protocol --degraded --protocols=nfs3 --long -i 5 -r 2 --nodes all --orderby numops --totalby op >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log

        echo "isi_for_array -s uptime" >> /ifs/admin/scripts/stat/stats/current.log
        isi_for_array -s uptime >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo >> /ifs/admin/scripts/stat/stats/current.log
        echo "isi_for_array -s top -n 3" >> /ifs/admin/scripts/stat/stats/current.log
        isi_for_array -s top -n 3 >> /ifs/admin/scripts/stat/stats/top.$(date +%d%b%Y).log
        cat /ifs/admin/scripts/stat/stats/current.log >> /ifs/admin/scripts/stat/stats/stat.$(date +%d%b%Y).log

# grab heat stats
isi statistics heat --degraded --class write,namespace_write > /ifs/admin/scripts/stat/stats/heat_write_namespace_and_write_`date +%s`.txt

# clean up after yourself, and release your trap
        rm -f "$lockfile"
        trap - INT TERM EXIT
else
        echo "Lock Exists: $lockfile owned by $(cat $lockfile)"
fi