Stop Cloud instance in case there's no GPU activity

auto-stop-instance-gpu.sh

#!/bin/bash

# This script is used to stop Cloud instance in case there's no GPU activity for certain period of time
# Feel free to contribute https://gist.github.com/artoby/468a7ccbbbc35f6cc904ed668a189a45

idle_seconds_to_shutdown=3600
threshold=0
check_period=0.1
wait_before_shutdown=10
last_activity=$(date +%s)
previous_idle_seconds=0
idle_echo_period=5

while true
do

  gpu_utilization=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits)
  if (( $gpu_utilization > $threshold ))
  then
    last_activity=$(date +%s)
  fi

  idle_seconds=$(($(date +%s) - last_activity))
  if (( $idle_seconds > 0 ))
  then
    if (( $idle_seconds / $idle_echo_period != $previous_idle_seconds / $idle_echo_period ))
    then
      echo "$(date "+%F %T") Idling for $idle_seconds/$idle_seconds_to_shutdown seconds"
    fi
  fi
  previous_idle_seconds=$idle_seconds

  if (( $idle_seconds >= $idle_seconds_to_shutdown ))
  then
    echo "$(date "+%F %T") Shutting down in $wait_before_shutdown seconds"
    # wait a little bit more before actually pulling the plug
    sleep $wait_before_shutdown
    sudo poweroff
  fi

  sleep $check_period
done