CUDA next/all available GPUs

cuda_available_gpus.sh

# Returns the next or all available GPU id(s), or -1 if all are busy.
# Usage:
#
# One GPU:
#
# $ CUDA_VISIBLE_DEVICES=$(available_gpu) train.sh
# 
# All GPUs (comma-separated):
#
# $ CUDA_VISIBLE_DEVICES=$(available_gpu) train.sh

function available_gpu() {
  for i in $(nvidia-smi -L|sed -E 's/^GPU ([0-9]+).*/\1/')
  do
    no_process=$(nvidia-smi -i $i |grep 'No running processes found'|wc -l)
    if [ "$no_process" = "1" ]
    then
      echo $i
      return
    fi
  done
  echo "-1"
}

available_gpu

function available_gpus() {
  for i in $(nvidia-smi -L|sed -E 's/^GPU ([0-9]+).*/\1/')
  do
    no_process=$(nvidia-smi -i $i |grep 'No running processes found'|wc -l)
    if [ "$no_process" = "1" ]
    then
      echo -n "$i,"
    fi
  done
  echo
}

available_gpus

Python version: https://pypi.org/project/nvgpu/