Check if NFS mounted directory is stale

check-nfs.sh

#!/bin/bash

PATH=/bin:/usr/bin:/usr/local/bin

check-nfs () {
    local TMPFILE=/tmp/checknfs.$$ RET=0 ORPHAN SUBSHELLPID

    if [ "$#" -eq 0 ]; then
        cat<<EOF
usage: check-nfs NFS-DIRECTORY...

Check if accessing any of NFS-DIRECTORY failed

EOF
        return 1
    fi

    while [ -n "$1" ]; do
        read -t 1 < <(echo $BASHPID >"$TMPFILE"; stat -t "$1" 2>/dev/null)
        if [ "$?" -gt 128 ]; then
	    #echo "error: $1"
	    ORPHAN=$(cat $TMPFILE)
	    SUBSHELLPID=$(ps --ppid $ORPHAN -o pid=)
	    [ -n "$SUBSHELLPID" ] && kill -9 $SUBSHELLPID
	    kill -9 $ORPHAN
	    RET=1
        fi
        shift
        rm -f $TMPFILE
    done
    return "$RET"
}

check-nfs "$@"

The key point is, to execute some command with timeout using read.

The second important one is, the subshell executing stat is not terminated normally iff stat is hanged due to the stale NFS. We need to kill the subshell and stat process manually.

Found a link to here on stackoverflow and made this for my crontab to execute every minute:

#!/bin/bash
stat_timeout=0.1 # seconds
umount_timeout=1200 # seconds

# Force/Lazy unmount hung NFS filesystems
for mp in $(mount -t nfs4 | awk '{print $3}'); do
    timeout -s kill $stat_timeout stat -t "$mp" > /dev/null
    if [[ $? == 137 ]]; then
        umount -fl "$mp"
        #echo "Unmounting hung NFS: $mp" 2>
    fi
done

# Fork inotify auto-unmounters
for mp in $(mount -t nfs4 | awk '{print $3}'); do
    pgrep -f "inotifywait(.*)$mp" > /dev/null && continue
    ( inotifywait -qq -t$umount_timeout "$mp";
      [[ $? == 2 ]] &&
          umount "$mp"; ) &
done