build hyperdrive on setonix

hyperdrive_hip_setonix.sh

# salloc --nodes=1 --partition=gpu-highmem --account=pawsey0875-gpu -t 04:00:00 --gres=gpu:1 --exclusive

# if you want segfaults:
# ROCM_VER=system DEBUG=false RAYON_NUM_THREADS=7 ./hyperdrive_hip_setonix.sh

# ##### #
# SETUP #
# ##### #

set -eux
[ -z "$PAWSEY_PROJECT" ] && ( echo "PAWSEY_PROJECT not set"; exit 1 ) || echo "PAWSEY_PROJECT: $PAWSEY_PROJECT"
[ -z "$USER" ] && ( echo "USER not set"; exit 1 ) || echo "USER: $USER"
export PAWSEY=${PAWSEY:=setonix}; echo "PAWSEY: $PAWSEY"
[ -z "$MYSOFTWARE" ] && ( echo "MYSOFTWARE not set"; exit 1 ) || echo "MYSOFTWARE: $MYSOFTWARE"
[ -z "$MYSCRATCH" ] && ( echo "MYSCRATCH not set"; exit 1 ) || echo "MYSCRATCH: $MYSCRATCH"
export PACKAGE="hyperdrive"; echo "PACKAGE: $PACKAGE"
export PREFIX="${MYSOFTWARE}/${PACKAGE}"; echo "PREFIX: $PREFIX"
export ROCM_VER="${ROCM_VER:=system}"; echo "ROCM_VER: $ROCM_VER"
export RUST_VER="${RUST_VER:=stable}"; echo "RUST_VER: $RUST_VER"
export TMPDIR="${TMPDIR:=/tmp/${USER}}"; echo "TMPDIR: $TMPDIR"; mkdir -p $TMPDIR
export CARGO_HOME="${CARGO_HOME:=${TMPDIR}/.cargo-${USER}}"; echo "CARGO_HOME: $CARGO_HOME"; mkdir -p $CARGO_HOME
export RUSTUP_HOME="${RUSTUP_HOME:=${TMPDIR}/.rustup-${USER}}"; echo "RUSTUP_HOME: $RUSTUP_HOME"; mkdir -p $RUSTUP_HOME

[[ "$CARGO_HOME" == "/software/"* ]] || [[ "$RUSTUP_HOME" == "/software/"* ]] \
  && echo "trying to use /software for software? That's a paddlin'. try this: \
    ( unset CARGO_HOME RUSTUP_HOME; $0 )" && exit 1

# ####### #
# MODULES #
# ####### #

# load rocm, unload if already loaded
function load_rocm() {
  [ -z "$ROCM_VER" ] && export ROCM_VER=$1
  [ -z "$ROCM_VER" ] && export ROCM_VER=system
  if [[ ! -z $( set +eux; echo "$ROCM_PATH") ]]; then
    echo "ROCM_PATH already set: $ROCM_PATH . unloading";
    module unload rocm
    unset ROCM_PATH LIBCLANG_PATH HIPIFY_INCLUDE HIP_PATH
  fi
  if [[ -z $1 || $1 == "system" ]]; then
    echo "loading system rocm"
    module load rocm
  else
    # Credit: Pascal
    echo "load rocm $1 from Pawsey unsupported modules"
    local version=${1}
    local major_version=${version%.*}
    module use /software/setonix/unsupported
    module load rocm/${version}
    export PE_PKGCONFIG_LIBS=$(echo $PE_PKGCONFIG_LIBS | sed "s/rocm-${major_version}//g")
  fi
  [ -z "$ROCM_PATH" ] && (echo "ROCM_PATH not set after loading rocm $ROCM_VER"; return 1) || echo "ROCM_PATH: $ROCM_PATH"
  # [ -z "$CRAY_ROCM_INCLUDE_OPTS" ] && (echo "CRAY_ROCM_INCLUDE_OPTS not set after loading rocm $ROCM_VER"; return 1) || echo "CRAY_ROCM_INCLUDE_OPTS: $CRAY_ROCM_INCLUDE_OPTS"
  export LIBCLANG_PATH="${LIBCLANG_PATH:="$ROCM_PATH/llvm/lib"}"
  [ -d "$LIBCLANG_PATH" ] || (echo "LIBCLANG_PATH not found: $LIBCLANG_PATH"; return 1)
  [ -x "$LIBCLANG_PATH/libclang.so" ] || (echo "libclang.so not found: $LIBCLANG_PATH/libclang.so"; return 1)
  export HIPIFY_INCLUDE="${HIPIFY_INCLUDE:="-I${ROCM_PATH}/include/hipify/"}"
  [ -d "${ROCM_PATH}/include/hipify/" ] || (echo "\${ROCM_PATH}/include/hipify/ not found: ${ROCM_PATH}/include/hipify/"; exit 1)
  [ -f "${ROCM_PATH}/include/hipify/stddef.h" ] || (echo "stddef.h not found in ${ROCM_PATH}/include/hipify/stddef.h"; exit 1)
  export HIP_PATH="${HIP_PATH:="${ROCM_PATH}/hip/"}"
  [ -d "${HIP_PATH}" ] || (echo "\${HIP_PATH} not found: ${HIP_PATH}"; exit 1)
  [ -x "${HIP_PATH}/bin/hipcc" ] || (echo "${HIP_PATH}/bin/hipcc not executable"; exit 1)
  [ -d "${HIP_PATH}/include/hip" ] || (echo "${HIP_PATH}/include/hip not found"; exit 1)
  echo "HIP_PATH: $HIP_PATH"
}
load_rocm ${ROCM_VER}

module load cmake/3.24.3
# load fontconfig from Pawsey hidden dependency modules (only needed for plotting feature)
module use /software/setonix/2023.08/modules/zen3/gcc/12.2.0/dependencies/
module load .fontconfig/2.13.94-rykl4uf .libpng/1.6.37-5rosdzt

[ -z "$ROCM_PATH" ] && (echo "ROCM_PATH not set after loading rocm $ROCM_VER"; exit 1) || echo "ROCM_PATH: $ROCM_PATH"
[ -z "$CRAY_ROCM_INCLUDE_OPTS" ] && (echo "CRAY_ROCM_INCLUDE_OPTS not set after loading rocm $ROCM_VER"; exit 1) || echo "CRAY_ROCM_INCLUDE_OPTS: $CRAY_ROCM_INCLUDE_OPTS"
export LIBCLANG_PATH="$ROCM_PATH/llvm/lib"
[ -d "$LIBCLANG_PATH" ] || (echo "LIBCLANG_PATH not found: $LIBCLANG_PATH"; exit 1)
[ -x "$LIBCLANG_PATH/libclang.so" ] || (echo "libclang.so not found: $LIBCLANG_PATH/libclang.so"; exit 1)
[ -d "${ROCM_PATH}/include/hipify/" ] || (echo "\${ROCM_PATH}/include/hipify/ not found: ${ROCM_PATH}/include/hipify/"; exit 1)
[ -f "${ROCM_PATH}/include/hipify/stddef.h" ] || (echo "stddef.h not found in ${ROCM_PATH}/include/hipify/stddef.h"; exit 1)
export HIPIFY_INCLUDE="-I${ROCM_PATH}/include/hipify/"


# ########### #
# ASTRO STUFF #
# ########### #

export MWA_BEAM_FILE="$MYSOFTWARE/mwa_full_embedded_element_pattern.h5"
[ -f $MWA_BEAM_FILE ] || wget -O "$MWA_BEAM_FILE" $'http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5'

# ### #
# GIT #
# ### #

echo "clone hyperdrive to \$MYSOFTWARE/$PACKAGE/$PACKAGE-git"
cd "$PREFIX"
[ -d "${PACKAGE}-git" ] || git clone --branch setonix https://github.com/MWATelescope/mwa_hyperdrive.git ${PACKAGE}-git
cd "${PACKAGE}-git"

# #### #
# RUST #
# #### #

# install rust
[ -x "$(command -v rustup)" ] \
  || curl https://sh.rustup.rs -sSf | env CARGO_HOME=$CARGO_HOME RUSTUP_HOME=$RUSTUP_HOME sh \
      -s -- -y --no-modify-path --profile minimal --default-toolchain $RUST_VER
[ -x "$(command -v cargo)" ] || source "$CARGO_HOME/env"
[ -x "$(command -v cargo)" ] || (echo "cargo not found"; exit 1)
rustup override set $RUST_VER # temporary override for this dir

# ####### #
# BINDGEN #
# ####### #

[ -x "$(command -v bindgen)" ] || cargo install bindgen-cli
"${PREFIX}/${PACKAGE}-git/src/gpu/update_rust_bindings.sh" || echo "bindgen fail, meh. $?"

# #### #
# TEST #
# #### #

rm -rf "${PREFIX}/${PACKAGE}-git/target" || true
export HYPERBEAM_HIP_ARCH="gfx90a"

export profile=release target=release
export DEBUG=${DEBUG:=false}
if [ "$DEBUG" != "false" ]; then
  export profile=dev target=debug
  export HIPCC_VERBOSE=1
  export RUST_BACKTRACE=full
fi

export features="--features=hdf5-static,cfitsio-static,hip"

set +u
export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:=1}
eval cargo +$RUST_VER test \
  --profile $profile \
  $features \
  $1 -- --nocapture --test-threads $RAYON_NUM_THREADS $2