oneCCL build and run benchmark sunspot

build_one_ccl_examples.sh

#!/bin/bash -l
#PBS -A Aurora_deployment
#PBS -l filesystems=home:gila
#PBS -l select=2
#PBS -l place=scatter
#PBS -l walltime=00:15:00
#PBS -q workq

#PBS -j oe
#PBS -k doe

COPY_DIR="${PWD}/one_ccl_examples"

# Copy the files over, if they don't exist
if [ ! -d "$COPY_DIR" ]; then
    cp -r /soft/compilers/oneapi/2024.04.15.001/oneapi/ccl/latest/share/doc/ccl/examples $COPY_DIR
fi

cd $COPY_DIR
module use /soft/preview-modulefiles/24.086.0
module load frameworks/2024.04.15.002
module load intel_compute_runtime/release/803.29
module load cmake

export CC=$(which icx)
echo "CC=${CC}"
export CXX=$(which icpx)
echo "CXX=${CXX}"
export DPCPP_ROOT=$(realpath $(dirname $CC)/..)
echo "DPCPP_ROOT=${DPCPP_ROOT}"

echo "*******************************************************"
echo "Building with modules:"
module list
echo "*******************************************************"
cmake -S . -B build -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DCOMPUTE_BACKEND=dpcpp \
    -DCMAKE_INSTALL_PREFIX=$(pwd)/build/_install && cmake --build build -j $(nproc) -t install

cd ..

# Launch the benchmark, if in an allocation
if [ -d "$PBS_NODEFILE" ]; then
    NNODES=`wc -l < $PBS_NODEFILE`
    NPROC_PER_NODE=12
    let NRANKS=${NNODES}*${NPROC_PER_NODE}

    # First test on one node
    echo "*******************************************************"
    echo "Launching ${NPROC_PER_NODE} processes on one node"
    echo "Which mpiexec? $(which mpiexec)"
    echo "*******************************************************"
    export CCL_PROCESS_LAUNCHER=pmix
    mpiexec -np $NPROC_PER_NODE -ppn $NPROC_PER_NODE --pmi=pmix "${COPY_DIR}/benchmark/benchmark"

    # And then on all nodes
    echo "*******************************************************"
    echo "Launching ${NRANKS} processes on ${NNODES} nodes"
    echo "Which mpiexec? $(which mpiexec)"
    echo "*******************************************************"
    mpiexec -np $NRANKS -ppn $NPROC_PER_NODE --pmi=pmix "${COPY_DIR}/benchmark/benchmark"
fi


# Cleanup
rm -rf $COPY_DIR