Run parallel jobs with tmux

run_tmux_parallel.sh

# Run multiple commands in a tmux session
script_dir=$(dirname "$(realpath -s "$0")")
if [[ $# -ne 1 ]]; then
	sess=session_name
else
	sess="$1"
fi

tmux new -d -s "$sess" -c "$script_dir"   # Use default directory as this script directory

for window in {0..2}; do
	# Window 0 or 1 may already exist so it will print error. Ignore that.
	if ! tmux list-pane -t "$sess:$window" &> /dev/null; then
		tmux new-window -c "$script_dir" -t "$sess:$window"
	fi
	command="CUDA_VISIBLE_DEVICES=$window python train.py --arg $((window+1))"
	tmux send-keys -t "$sess:$window" "$command" Enter
done

echo
echo "Tmux session $sess created. Run 'tmux attach -t $sess' to attach to the session."

tmux_parallel_status.sh

if [[ $# -ne 1 ]]; then
	sess=session_name
else
	sess="$1"
fi

echo "Tmux session: $sess"

all_pane_pids=""
for window in {0..2}; do
	# Collect all pane pids
	pane_pid="$(tmux list-panes -t "$sess:$window" -F "#{pane_pid}")"
	all_pane_pids="$all_pane_pids $pane_pid"
done

num_pids=$(echo "$all_pane_pids" | wc -w)
num_running=0
for pid in $all_pane_pids; do
	# Get child pid of shell
	child_pid=$(ps -ef | awk -v pid=$pid '$3==pid {print $2}')

	# count how many processes are running
	if [[ -n "$child_pid" ]] && [[ -d /proc/$child_pid ]]; then
		((num_running++))
	fi
done
num_finished=$((num_pids-num_running))

echo "Number of panes: $num_pids"
echo "Number of running: $num_running"
echo "Number of finished: $num_finished"
echo "Percent finished: $(echo "scale=2; $num_finished/$num_pids*100" | bc)%"