distributed run script for tray_rust

run_distributed.sh

#PBS -l nodes=22:ppn=32,pmem=2GB,walltime=00:10:00
#PBS -N tray_rust
# It's assumed all binaries and files are on a shared filesystem
PROG=./target/release/tray_rust_wopr
WORK_DIR=/home/sci/will/Repos/tray_rust/
DOWNLOADS=/home/sci/will/Downloads/
SCENE=${DOWNLOADS}/material_test_scene.json
OUTPUT=${DOWNLOADS}/plastic.png
KEY=<path to ssh private key>

cd $WORK_DIR
unique_hosts=`uniq $PBS_NODEFILE`
echo $unique_hosts

# This script sshs onto each machine we've been allocated
# and starts the worker running in the background. Note: PBS won't
# kill the jobs started on machines that aren't the master if the master
# job is killed
for host in $unique_hosts
do
	echo "Starting worker on $host"
	ssh -i $KEY $host WORK_DIR=$WORK_DIR PROG=$PROG HOST=$host 'bash -s' <<-'END'
		cd $WORK_DIR
		nohup $PROG --worker -n 32 > ./worker_output/${HOST}.txt 2>&1 &
	END
	echo "ssh status for $host $?"
done

# Wait a little bit for the processes to get running and be listening for the master
sleep 15

echo "Master is on `hostname`"
unique_hosts=`echo $unique_hosts | sed "s/\n//g"`
# Now launch the master on this host
echo "--master $unique_hosts"
$PROG $SCENE --master $unique_hosts -o $OUTPUT \
	| tee ./batch_run/wopr_live_out.txt