Compiles and runs hyperdrive on setonx. Additional notes for ROCm development.

hyperdrive_cpu_setonix.sh

#!/bin/bash
#SBATCH -J "hypertest"
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --account=pawsey0875-gpu
#SBATCH --partition=gpu
#SBATCH --time=1:00:00
#SBATCH --exclusive
#SBATCH --gres=gpu:1
###SBATCH --array=0-1

# ####### #
# PRELUDE #
# ####### #

# This script should install hyperdrive (CPU only) and run a minimal visibility simulation test.
# It is based on a work in progress script for the GPU version.
# I'll leave the GPU stuff in because it's going to be useful for your future GPU work.
# Best of luck!

# Tip: You can run this in a salloc, on any of the gpu partitions.
#
# salloc --nodes=1 --partition=gpu-highmem --account=pawsey0875-gpu -t 04:00:00 --gres=gpu:1 --exclusive

# Tip: If you prefer singularity, you can use these containers.
# module load singularity/3.11.4-slurm
# export ROCM_VER="5.6.0"
# singularity exec --cleanenv --bind $PWD --bind /software docker://quay.io/pawsey/rocm-mpich-base:rocm${ROCM_VER}-mpich3.4.3-ubuntu22 /bin/bash

# ##### #
# SETUP #
# ##### #

[ -z "$PAWSEY_PROJECT" ] && ( echo "PAWSEY_PROJECT not set"; exit 1 ) || echo "PAWSEY_PROJECT: $PAWSEY_PROJECT"
[ -z "$USER" ] && ( echo "USER not set"; exit 1 ) || echo "USER: $USER"
export PAWSEY=${PAWSEY:=setonix}; echo "PAWSEY: $PAWSEY"
[ -z "$MYSOFTWARE" ] && ( echo "MYSOFTWARE not set"; exit 1 ) || echo "MYSOFTWARE: $MYSOFTWARE"
[ -z "$MYSCRATCH" ] && ( echo "MYSCRATCH not set"; exit 1 ) || echo "MYSCRATCH: $MYSCRATCH"
export PACKAGE="hyperdrive"; echo "PACKAGE: $PACKAGE"
export PREFIX="${MYSOFTWARE}/${PACKAGE}"; echo "PREFIX: $PREFIX"
export ROCM_VER="${ROCM_VER:=5.6.1}"; echo "ROCM_VER: $ROCM_VER"
export RUST_VER="${RUST_VER:=stable}"; echo "RUST_VER: $RUST_VER"
export TMPDIR="${TMPDIR:=/tmp/${USER}}"; echo "TMPDIR: $TMPDIR"; mkdir -p $TMPDIR
export CARGO_HOME="${CARGO_HOME:=${TMPDIR}/.cargo-${USER}}"; echo "CARGO_HOME: $CARGO_HOME"; mkdir -p $CARGO_HOME
export RUSTUP_HOME="${RUSTUP_HOME:=${TMPDIR}/.rustup-${USER}}"; echo "RUSTUP_HOME: $RUSTUP_HOME"; mkdir -p $RUSTUP_HOME

[[ "$CARGO_HOME" == "/software/"* ]] || [[ "$RUSTUP_HOME" == "/software/"* ]] \
  && echo "trying to use /software for software? How silly of you. Try this: \
    ( unset CARGO_HOME RUSTUP_HOME; $0 )" && exit 1

set -eux

# ####### #
# MODULES #
# ####### #

[ -z $(command -v cmake) ] && module load cmake/3.24.3
# load dependencies from Pawsey hidden dependency modules to avoid compiling these in spack.
# fontconfig and libpng only needed for hyperdrive's plotting feature.
module use /software/setonix/2023.08/modules/zen3/gcc/12.2.0/dependencies/
module load .fontconfig/2.13.94-rykl4uf .libpng/1.6.37-5rosdzt

# ######## #
# GPU MODS #
# ######## #
# - this is not needed for cpu version, but handy to know.
# - some environment variables are needed for rocm and bindgen

# function load_unsupported_rocm()
# {
#   echo "-> load rocm $1 from Pawsey unsupported modules"
#   if [ -z $1 ]; then
#     echo "Must pass rocm version"
#     exit 1
#   fi
#   local version=${1}
#   local major_version=${version%.*}
#   module use /software/setonix/unsupported
#   module load rocm/${version}
#   export PE_PKGCONFIG_LIBS=$(echo $PE_PKGCONFIG_LIBS | sed "s/rocm-${major_version}//g")
# }
# [ -z $( set +eux; echo "$ROCM_PATH") ] && load_unsupported_rocm ${ROCM_VER}

# [ -z "$ROCM_PATH" ] && (echo "ROCM_PATH not set after loading rocm $ROCM_VER"; exit 1) || echo "ROCM_PATH: $ROCM_PATH"
# [ -z "$CRAY_ROCM_INCLUDE_OPTS" ] && (echo "CRAY_ROCM_INCLUDE_OPTS not set after loading rocm $ROCM_VER"; exit 1) || echo "CRAY_ROCM_INCLUDE_OPTS: $CRAY_ROCM_INCLUDE_OPTS"
# export LIBCLANG_PATH="$ROCM_PATH/llvm/lib"
# [ -d "$LIBCLANG_PATH" ] || (echo "LIBCLANG_PATH not found: $LIBCLANG_PATH"; exit 1)
# [ -x "$LIBCLANG_PATH/libclang.so" ] || (echo "libclang.so not found: $LIBCLANG_PATH/libclang.so"; exit 1)
# [ -d "${ROCM_PATH}/include/hipify/" ] || (echo "\${ROCM_PATH}/include/hipify/ not found: ${ROCM_PATH}/include/hipify/"; exit 1)
# [ -f "${ROCM_PATH}/include/hipify/stddef.h" ] || (echo "stddef.h not found in ${ROCM_PATH}/include/hipify/stddef.h"; exit 1)
# export HIPIFY_INCLUDE="-I${ROCM_PATH}/include/hipify/"

# ########### #
# ASTRO STUFF #
# ########### #
# - minimal idempotent test data
export obsid=1087251016
export outdir="${MYSCRATCH}/${obsid}"
mkdir -p $outdir
export metafits="${outdir}/${obsid}.metafits"
[ -f "$metafits" ] || wget -O "$metafits" $'http://ws.mwatelescope.org/metadata/fits?obs_id='${obsid}
export hyp_srclist="${outdir}/srclist_${obsid}.yaml"
[ -f "$hyp_srclist" ] || cat >$hyp_srclist <<EOF
TEST:
- ra: 0
  dec: -27
  comp_type: point
  flux_type:
    list:
    - freq: 100_000_000
      i: 1
    - freq: 300_000_000
      i: 1
EOF
export MWA_BEAM_FILE="$MYSOFTWARE/mwa_full_embedded_element_pattern.h5"
[ -f $MWA_BEAM_FILE ] || wget -O "$MWA_BEAM_FILE" $'http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5'
export hyp_toml="${outdir}/hyp_conf.toml"
[ -f "$hyp_toml" ] || rm "$hyp_toml"

# did you know: hyperdrive can take config from a .toml file?
cat >$hyp_toml <<EOF
[beam]
# uncomment to make segfaults magically disappear before your very eyes!
# no_beam = true
unity_dipole_gains = false

[model]
no_precession = false
cpu = false

[sky-model]
source_list = "$hyp_srclist"
num_sources = 1

[vis-simulate]
metafits = "$metafits"
filter_points = false
filter_gaussians = false
filter_shapelets = false
num_timesteps = 1
ignore_dut1 = false
num_fine_channels = 1
middle_freq = 200000000.0
freq_res =      1280000.0
EOF

# ### #
# GIT #
# ### #
# clone hyperdrive

echo "clone hyperdrive to \$MYSOFTWARE/$PACKAGE/$PACKAGE-git"
cd "$PREFIX"
[ -d "${PACKAGE}-git" ] || git clone https://github.com/MWATelescope/mwa_hyperdrive.git ${PACKAGE}-git
cd "${PACKAGE}-git"

# ##### #
# PATCH #
# ##### #

# stupid extra flags for the stupid hip compiler
# sed -i  's/let mut hip_target = cc::Build::new();/let mut hip_target = cc::Build::new() ; hip_target.flag("--offload-arch=gfx90a");/' build.rs

# #### #
# RUST #
# #### #

# install rust
[ -x "$(command -v rustup)" ] \
  || curl https://sh.rustup.rs -sSf | env CARGO_HOME=$CARGO_HOME RUSTUP_HOME=$RUSTUP_HOME sh \
      -s -- -y --no-modify-path --profile minimal --default-toolchain $RUST_VER
[ -x "$(command -v cargo)" ] || source "$CARGO_HOME/env"
[ -x "$(command -v cargo)" ] || (echo "cargo not found"; exit 1)
rustup override set $RUST_VER # temporary override for this dir

# ####### #
# BINDGEN #
# ####### #
# - this is needed for ensuring the gpu types are correctly bound to rust types.

# [ -x "$(command -v bindgen)" ] || cargo install bindgen-cli
# "${PREFIX}/${PACKAGE}-git/src/gpu/update_rust_bindings.sh" || (echo "bindgen fail $?"; exit 1)

# ####### #
# COMPILE #
# ####### #

# clear the target dir, just in case
rm -rf "${PREFIX}/${PACKAGE}-git/target" || true

# the following is what I've been (unsucessfully) trying for hyperdrive's GPU version.
# - --profile dev turns off compiler optimizations, needed for ROCm
# eval cargo +$RUST_VER build \
#     --profile dev \
#     --no-default-features --features=hdf5-static,cfitsio-static,hip \
#     --verbose || exit "compile fail"

# here is the cpu version, production build. (plot is optional)
eval cargo +$RUST_VER build \
    --no-default-features --features=hdf5-static,cfitsio-static,plotting \
    --verbose || exit "compile fail"

# ##### #
# DEBUG #
# ##### #
# this is how i'm debuggint eh GPU version.
# cat > rocgdbinit <<EOF
# set amdgpu precise-memory on
# set breakpoint pending on
# break src/gpu/model.cu:401
# commands
#   set \$i=0
#   while \$i<64
#     lane \$i
#     print i_unf_t1
#     set \$i=\$i+1
#   end
#   cont
# end
# run
# EOF

# export hyperdrive="rocgdb -x rocgdbinit --args ${PREFIX}/${PACKAGE}-git/target/debug/hyperdrive"
export hyperdrive="${PREFIX}/${PACKAGE}-git/target/debug/hyperdrive"
eval $hyperdrive vis-simulate -vvv -- "$hyp_toml"
    # 2>&1 | tee "${outdir}/log/hyp_vis-sim.log"

# echo ${PIPESTATUS[@]}
# ret=${PIPESTATUS[0]}
# exit $ret