Check free memory on all GPUs across multiple servers together (instead of cumbersome sequential login-checks). Custom-made for CS servers @ UVa

gpu_check.sh

#!/bin/bash
read -s password
myfun()
{
        f="$(mktemp /tmp/check.XXXXXX)"
        nvidia-smi -q -x > $f
        python -c "import xml.etree.ElementTree as ET; tree = ET.parse('$f'); root = tree.getroot(); import os; myname =  os.uname()[1].split('.')[0]; z = [myname+'['+child.find('minor_number').text+'] : '+child.find('fb_memory_usage').find('free').text+'/'+child.find('fb_memory_usage').find('total').text for child in root.findall('gpu')]; print('\n'.join(z) + '\n')"
}
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv01.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv02.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv03.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv04.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv05.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv06.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv07.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv08.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv09.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv10.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv11.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv12.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv13.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv14.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv15.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv16.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv17.cs.virginia.edu "$(typeset -f); myfun" &
sshpass -p $password ssh -o StrictHostKeyChecking=no $1@gpusrv18.cs.virginia.edu "$(typeset -f); myfun"
wait