Example SLURM script to be used on Stampede. This script can be submitted with "sbatch sample.job". It runs the application on two nodes, using 16 cores per node.

config.sh

#!/bin/bash

if [ -f CMakeCache.txt ]; then
    rm -rf CMakeCache.txt
fi

cmake -DHPX_NO_INSTALL=True                                                     \
      -DCMAKE_AR=$ICC_BIN/xiar                                                  \
      -DCMAKE_CXX_COMPILER=$ICC_BIN/icpc                                        \
      -DCMAKE_C_COMPILER=$ICC_BIN/icc                                           \
      -DCMAKE_BUILD_TYPE=Release                                                \
      -DBOOST_ROOT=$WORK/boost/1_53_0                                           \
      -DBOOST_SUFFIX="-il-mt-1_53"                                              \
      -DIBVERBS_ROOT=/opt/ofed                                                  \
      -DRDMACM_ROOT=/opt/ofed                                                   \
      -DHPX_HAVE_PARCELPORT_MPI=True                                            \
      -DTBB_ROOT=$TBBROOT                                                       \
      -DHPX_MALLOC="tbbmalloc"                                                  \
      -Wdev                                                                     \
      $WORK/hpx

run_nbody_hpx_weak_host.sh

#!/bin/bash

WORK=/work/02405/heller
APP="$WORK/build/nbody_demo/release-host/bin/nbody_hpx_weak"
APP="$APP -Ihpx.parcel.mpi.enable=1"
APP="$APP -Ihpx.parcel.mpi.max_requests!=2147483647"
APP="$APP -Ihpx.parcel.tcpip.enable=0"
APP="$APP -Ihpx.stacks.use_guard_pages=0"
APP="$APP -Inbody.hostUpdateGroups!=1"
APP="$APP -ILibGeoDecomp.asyncThreshold!=2000"
PERFCTRS="--hpx:print-counter /agas{locality#*/total}/count/cache-evictions"
PERFCTRS="$PERFCTRS --hpx:print-counter /agas{locality#*/total}/count/cache-hits"
PERFCTRS="$PERFCTRS --hpx:print-counter /agas{locality#*/total}/count/cache-insertions"
PERFCTRS="$PERFCTRS --hpx:print-counter /agas{locality#*/total}/count/cache-misses"
PERFCTRS="$PERFCTRS --hpx:print-counter /serialize{locality#*/total}/count/mpi/received"
PERFCTRS="$PERFCTRS --hpx:print-counter /serialize{locality#*/total}/count/mpi/sent"
PERFCTRS="$PERFCTRS --hpx:print-counter /serialize{locality#*/total}/time/mpi/received"
PERFCTRS="$PERFCTRS --hpx:print-counter /serialize{locality#*/total}/time/mpi/sent"
PERFCTRS="$PERFCTRS --hpx:print-counter /parcels{locality#*/total}/count/mpi/received"
PERFCTRS="$PERFCTRS --hpx:print-counter /parcels{locality#*/total}/count/mpi/sent"
#PERFCTRS="$PERFCTRS --hpx:print-counter /parcels{locality#*/total}/time/mpi/received"
#PERFCTRS="$PERFCTRS --hpx:print-counter /parcels{locality#*/total}/time/mpi/sent"
PERFCTRS="$PERFCTRS --hpx:print-counter /messages{locality#*/total}/count/mpi/received"
PERFCTRS="$PERFCTRS --hpx:print-counter /messages{locality#*/total}/count/mpi/sent"
#PERFCTRS="$PERFCTRS --hpx:print-counter /messages{locality#*/total}/time/mpi/received"
#PERFCTRS="$PERFCTRS --hpx:print-counter /messages{locality#*/total}/time/mpi/sent"
PERFCTRS="$PERFCTRS --hpx:print-counter /data{locality#*/total}/count/mpi/received"
PERFCTRS="$PERFCTRS --hpx:print-counter /data{locality#*/total}/count/mpi/sent"
PERFCTRS="$PERFCTRS --hpx:print-counter /data{locality#*/total}/time/mpi/received"
PERFCTRS="$PERFCTRS --hpx:print-counter /data{locality#*/total}/time/mpi/sent"
PERFCTRS="$PERFCTRS --hpx:print-counter /runtime{locality#*/total}/memory/resident"
PERFCTRS="$PERFCTRS --hpx:print-counter /runtime{locality#*/total}/memory/virtual"
#PERFCTRS="$PERFCTRS --hpx:print-counter-interval 20"

export SPEED=1.0

numactl --interleave=0,1 $APP $PERFCTRS

sample.job

#!/bin/bash
#SBATCH -J nbody_weak_hpx_2
#SBATCH -o nbody_weak_hpx_2.o%j
#SBATCH -e nbody_weak_hpx_2.e%j
#SBATCH -p normal
#SBATCH -N 2
#SBATCH -n 2
#SBATCH -t 01:00:00

APP_HOST="/work/02405/heller/nbody_demo/stampede/run_nbody_hpx_weak_host.sh"
ibrun $APP_HOST