Reprocude ecTrans + ACFL + mcpu=native crash

reproduce.sh

#!/usr/bin/env bash

#SBATCH -t 0:10:00
#SBATCH -N 1
#SBATCH --exclusive
#SBATCH --job-name="_ECTrans"
#SBATCH -o %x_%j_%N.slurm.out

# reload env
[ -f ~/.bashrc ] && . ~/.bashrc

set -eu -o pipefail

_ectrans_compiler_flags=${1:-normal}

ECTRANS_SBATCH_SCRIPT_DIR=${ECTRANS_SBATCH_SCRIPT_DIR:-$(dirname $(readlink -f $0))}
DATE_FMT=${DATE_FMT:-now}

##
## Sources (use absolute paths)
##
export ectrans_SRC=${ectrans_SRC:-${PWD}/ectrans/}
export ecbuild_ROOT=${ecbuild_ROOT:-${PWD}/ecbuild/}
export fiat_SRC=${fiat_SRC:-${PWD}/fiat/}

[ ! -d $ectrans_SRC ] && git clone https://github.com/ecmwf-ifs/ectrans.git $ectrans_SRC
[ ! -d $fiat_SRC ] && git clone https://github.com/ecmwf-ifs/fiat.git $fiat_SRC
[ ! -d $ecbuild_ROOT ] && git clone https://github.com/ecmwf/ecbuild.git $ecbuild_ROOT

OPENBLAS_VERSION=0.3.23
FFTW_VERSION=3.3.10

UNAMEM=$(uname -m)
UNAMES=$(uname -s)

WORK_DIR=${ECTRANS_SBATCH_SCRIPT_DIR}/test_$_ectrans_compiler_flags/
BUILD_DIR=${WORK_DIR}/build
RUN_DIR=${WORK_DIR}/run
mkdir -p ${BUILD_DIR}
mkdir -p ${RUN_DIR}

##
## Build & prefix dirs
##
export fiat_BUILD=${BUILD_DIR}/fiat_build/
export ectrans_BUILD=${BUILD_DIR}/ectrans_build/

export fiat_ROOT=${BUILD_DIR}/fiat_prefix/
export ectrans_ROOT=${BUILD_DIR}/ectrans_prefix/

echo " -- setup compiler env"
##
## Edit the following lines to match your environment
##
COMPILER="acfl:23.04.1"
_BULL_SOFTWARE_DIR=~/software
source ${_BULL_SOFTWARE_DIR}/compilers.sh ${COMPILER} YES NO

case $_ectrans_compiler_flags in
    normal)
        export CFLAGS="-O3 -DNDEBUG"
        export FCFLAGS="$CFLAGS"
        export CXXFLAGS="$CXXFLAGS"
        _ECTRANS_BUILD_TYPE="None"
        ;;
    mcpu)
        export CFLAGS="-O3 -mcpu=native -DNDEBUG"
        export FCFLAGS="$CFLAGS"
        export CXXFLAGS="$CXXFLAGS"
        _ECTRANS_BUILD_TYPE="None"
        ;;
    *) echo "Error: unsupported flag mode '$_ectrans_compiler_flags'" && exit 1 ;;
esac

if [ ! -x ${fiat_ROOT}/bin/fiat ]; then
    rm -rf ${fiat_BUILD}
    mkdir ${fiat_BUILD}
    (cd ${fiat_BUILD} && cmake \
        -DCMAKE_BUILD_TYPE="$_ECTRANS_BUILD_TYPE" -DCMAKE_Fortran_FLAGS="$FCFLAGS" -DCMAKE_C_FLAGS="$CFLAGS" \
        -DCMAKE_INSTALL_PREFIX=${fiat_ROOT} \
        -DENABLE_TESTS=OFF \
        ${fiat_SRC}) \
        |& tee ${BUILD_DIR}/01_fiat_cmake.log

    (cd ${fiat_BUILD} && make -j $(nproc) VERBOSE=1) |& tee ${BUILD_DIR}/02_fiat_make.log
    rm -rf ${fiat_ROOT}
    mkdir ${fiat_ROOT}
    (cd ${fiat_BUILD} && make install) |& tee ${BUILD_DIR}/03_fiat_make_install.log
else
    echo "##-- Fiat already built"
fi

${fiat_ROOT}/bin/fiat --version |& tee ${BUILD_DIR}/07_run.log
${fiat_ROOT}/bin/fiat --git |& tee -a ${BUILD_DIR}/07_run.log
${fiat_ROOT}/bin/fiat --info |& tee -a ${BUILD_DIR}/07_run.log

if [ ! -x ${ectrans_ROOT}/bin/ectrans ]; then
    rm -rf ${ectrans_BUILD}
    mkdir ${ectrans_BUILD}
    (cd ${ectrans_BUILD} && cmake \
        -DCMAKE_BUILD_TYPE="$_ECTRANS_BUILD_TYPE" -DCMAKE_Fortran_FLAGS="$FCFLAGS" -DCMAKE_C_FLAGS="$CFLAGS" \
        -DCMAKE_INSTALL_PREFIX=${ectrans_ROOT} \
        -DENABLE_TESTS=ON \
        ${ectrans_SRC}) \
        |& tee ${BUILD_DIR}/04_ectrans_cmake.log

    (cd ${ectrans_BUILD} && make -j $(nproc) VERBOSE=1) |& tee ${BUILD_DIR}/05_ectrans_make.log
    rm -rf ${ectrans_ROOT}
    mkdir ${ectrans_ROOT}
    (cd ${ectrans_BUILD} && make install) |& tee ${BUILD_DIR}/06_ectrans_make_install.log
else
    echo "##-- ecTrans already built"
fi

${ectrans_ROOT}/bin/ectrans --version |& tee -a ${BUILD_DIR}/07_run.log
${ectrans_ROOT}/bin/ectrans --git |& tee -a ${BUILD_DIR}/07_run.log
${ectrans_ROOT}/bin/ectrans --info |& tee -a ${BUILD_DIR}/07_run.log


## MPI configuration
# export DR_HOOK_ASSERT_MPI_INITIALIZED=0
##TODO check DR_HOOK @ A64FX
# disable DR_HOOK globally
# export DR_HOOK=-1
export UCX_TLS=self,cma,ud_verbs
export OMPI_MCA_btl='^openib'
export NO_STOP_MESSAGE=yes
## Memory limit
# ulimit -s unlimited
## Distribution
##TODO revoir distribution sur NUMA
export OMP_NUM_THREADS=4
# export OMP_PROC_BIND=spread
export MPI_NUM_RANKS=1
## Misc
export LD_LIBRARY_PATH=${ectrans_BUILD}/lib:${LD_LIBRARY_PATH:-}

FILE=09_ectrans_run_dp.log
BINARY=ectrans-benchmark-dp
PROFILEARGS="--meminfo --norms"
ARGS="-n 1 -f 5 -l 40 --vordiv"
(echo "CMD: $BINARY $PROFILEARGS $ARGS"; cd ${ectrans_ROOT} \
    && mpirun -n $MPI_NUM_RANKS ./bin/$BINARY $PROFILEARGS $ARGS )\
    |& tee ${RUN_DIR}/$FILE