monitor.sh

#!/usr/bin/env bash
cd ${HOME}/mining/monitoring
module="eth"
set -e

GPU_LIST_USAGE=( `/opt/bin/nvidia-smi --query-gpu="utilization.gpu" --format=csv,noheader,nounits` )

total=0

for GPU in "${GPU_LIST_USAGE[@]}"
do
    total=$(( ${total} + ${GPU} ))
done

echo "Total ${total}"
if [ ${total} -eq 0 ]; then
   TIMESTAMP=`date +%Y%m%d%H%M%S`
   echo "${TIMESTAMP} respawn" >> error.log
   cd ${HOME}/${module}
   ./stop.sh
   mv ${HOME}/${module}/logs/stderr.log ${HOME}/${module}/logs/stderr.log.${TIMESTAMP}
   mv ${HOME}/${module}/logs/stdout.log ${HOME}/${module}/logs/stdout.log.${TIMESTAMP}
   sleep 3
   ./start.sh
else
   echo "process ok"
fi