check_gpu_users.sh

MEM_INFO=`nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader`

output=''

while read -r line
do
  PID=`echo $line | awk -F, '{print $1}'`
  MEM=`echo $line | awk -F, '{print $2}'`
  ELAPSED=`ps -p $PID -o etime --no-headers`
  UNAME=`ps -p $PID -o user --no-headers`

  dockerid=''
  dockermounts=''
  if [ $UNAME == "root" ]; then
    targetpid=$PID
    while [ $targetpid != 1 ]; do
      dockerid=`docker ps -q | xargs docker inspect --format '{{if eq .State.Pid '$targetpid'}}{{.ID}}{{ end }}' | tr -d '\n'`
      if [ ! -z "$dockerid" ]; then
        break
      fi
      targetpid=$(ps -o ppid= $targetpid | tr -d '\n' | tr -d ' ')
    done
  fi

  if [ ! -z "$dockerid" ]; then
    dockermounts=`docker inspect --format '{{ range .Mounts }} {{ .Source }}{{ end }}' $dockerid`
  fi

  output=$output$'\n'"pid: $PID, Elapsed time: $ELAPSED,  MEM: $MEM, User: $UNAME, Docker mounts (if any): $dockermounts"
done <<< "$MEM_INFO"
echo "$output" | sort