SLURM job script for 4 concurrent MPI runs, over 4 GPUs each, of fbpic

slurm_job_script.sh

#!/bin/bash
#SBATCH --partition=gpu
#SBATCH -t 72:00:00
#SBATCH --ntasks=16
#SBATCH --ntasks-per-node=16
#SBATCH --cpus-per-task=1
#SBATCH --mem-per-cpu=31200m
#SBATCH --gres=gpu:16
#SBATCH --gres-flags=enforce-binding
#SBATCH --account=berceanu_a+

module use $HOME/MyModules
module load miniforge3pic/latest

export FBPIC_DISABLE_THREADING=1
export MKL_NUM_THREADS=1
export NUMBA_NUM_THREADS=1
export OMP_NUM_THREADS=1

srun --ntasks=4 --gres=gpu:4 --mpi=pmi2 python fbpic_script.py one/ &
srun --ntasks=4 --gres=gpu:4 --mpi=pmi2 python fbpic_script.py two/ &
srun --ntasks=4 --gres=gpu:4 --mpi=pmi2 python fbpic_script.py three/ &
srun --ntasks=4 --gres=gpu:4 --mpi=pmi2 python fbpic_script.py four/ &
wait

slurm_single_bundle.sh

#!/bin/bash
#SBATCH --partition=gpu
#SBATCH -t 72:00:00
#SBATCH --ntasks=4
#SBATCH --ntasks-per-node=4
#SBATCH --cpus-per-task=1
#SBATCH --mem-per-cpu=31200m
#SBATCH --gres=gpu:4
#SBATCH --gres-flags=enforce-binding
#SBATCH --account=berceanu_a+

module use $HOME/MyModules
module load miniforge3pic/latest

export FBPIC_DISABLE_THREADING=1
export MKL_NUM_THREADS=1
export NUMBA_NUM_THREADS=1
export OMP_NUM_THREADS=1

srun --mpi=pmi2 python fbpic_script.py one/

without_slurm.sh

#!/usr/bin/env bash

module use $HOME/MyModules
module load miniforge3pic/latest

export FBPIC_DISABLE_THREADING=1
export MKL_NUM_THREADS=1
export NUMBA_NUM_THREADS=1
export OMP_NUM_THREADS=1

export CUDA_DEVICE_ORDER=PCI_BUS_ID

export CUDA_VISIBLE_DEVICES=0,1,2,3
mpirun -n 4 python fbpic_script.py one/ &
sleep 5

export CUDA_VISIBLE_DEVICES=4,5,6,7
mpirun -n 4 python fbpic_script.py two/ &
sleep 5

export CUDA_VISIBLE_DEVICES=8,9,10,11
mpirun -n 4 python fbpic_script.py three/ &
sleep 5

export CUDA_VISIBLE_DEVICES=12,13,14,15
mpirun -n 4 python fbpic_script.py four/ &

wait