Script launching a SLURM sbatch jobs until it is completed. This can be useful in case the SLURM environment allows jobs to run only a limited amount of time.

slurm_loop_sbatch.sh

#!/bin/bash

# Here the job is relaunched if it was not 100% completed, that is until a specific file exists
# The --wait argument allows to hang the execution until the job execution ends, before resubmitting it if needed

# Set vars
NAME=$1
JOB_FILE="train_$NAME.sh"
TRAIN_FILE="runs/gen_MMD/TSD_$NAME/train_results.json"

# Loop job until training is done
while [ ! -f "$TRAIN_FILE" ];
do
    sbatch --wait "$JOB_FILE"
done