Quick test of engaging IB

mpi_basic.sbatch

#!/bin/bash
# sbatch -p sched_mit_darwin2 -N 2 -n 24 --mincpus=12 --cores-per-socket=12
cd /nfs/cnhlab001/cnh/projects/mpi-test/argonne-tests/barrier
scontrol -o show node=${SLURM_STEP_NODELIST} | awk '{print $1}' | awk -F= '{print $2}' | awk '{print $1" slots=12"}' > hf
module purge
module load gcc/9.3.0 
module load openmpi/4.0.5
module load ucx/1.9.0

cat > barrier.c <<EOF
#include <stdio.h>
#include <stdlib.h>
#include "mpi.h"

#define NUMBER_OF_TESTS 10

int main( argc, argv )
int argc;
char **argv;
{
    int          rank;
    int          size;
    double       t1, t2, tmin;
    int          j, k, nloop;

    char procName[MPI_MAX_PROCESSOR_NAME+1];
    int  pNLen;


    MPI_Init( &argc, &argv );

    MPI_Get_processor_name( procName, &pNLen );
    procName[pNLen] = '\0';
    MPI_Comm_rank( MPI_COMM_WORLD, &rank );
    printf("Rank %d on host \"%s\"\n",rank, procName );
    MPI_Comm_size( MPI_COMM_WORLD, &size );
    if (rank == 0 && size == 1) 
	printf( "Kind\tnp\ttime (sec)\n" );

    nloop = 1000;
    tmin  = 1000;
    for (k=0; k<NUMBER_OF_TESTS; k++) {
	MPI_Barrier( MPI_COMM_WORLD );
	t1 = MPI_Wtime();
	for (j=0; j<nloop; j++) {
	    MPI_Barrier( MPI_COMM_WORLD );
	}
	t2 = (MPI_Wtime() - t1) / nloop;
	if (t2 < tmin) tmin = t2;
    }

    if (rank == 0) {
	printf( "Barrier\t%d\t%f usecs\n", size, tmin*1.e6 );
    }

    MPI_Finalize( );
    return 0;
}
EOF

mpicc barrier.c 
mpirun --bind-to core --report-bindings --mca btl '^openib' --mca orte_base_help_aggregate 0 --mca opal_warn_on_missing_libcuda 0 --hostfile hf -np 24 ./a.out 

run_with_sbatch_command.sh

cat mpi_basic.sbatch | sbatch -p sched_mit_darwin2 -N 2 -n 24 --mincpus=12 --cores-per-socket=12