end2end.sh

#!/bin/zsh -eux

export obsid="1090359520"
export outdir="/data/dev"

## preprocessing settings
#### time resolution to average to in seconds
export timeres_s=2
#### frequency resolution to average to in kHz
export freqres_khz=40
#### edge width to flag in kHz
export edgewidth_khz=80
#### extra birli args if any
export birli_args=""

## calibration settings
export hyp_srclist_args='--source-dist-cutoff=90 --veto-threshold 0.005'
export hyp_dical_args="--max-iterations 300 --stop-thresh 1e-20"
export dical_name="30l_src4k_"
export dical_args="--uvw-min 30l -n 4000 ${hyp_srclist_args} ${hyp_dical_args}"

## apply settings
export apply_name="2s_40kHz_trunc"
export apply_args="--time-average 2s --freq-average 40kHz --timesteps 32 33 34 35"

## peel settings
export ionosub_name="i1000"
export ionosub_args="--iono-sub 1000"

export OPENBLAS_NUM_THREADS=1

mkdir -p "${outdir}/${obsid}/raw"
export metafits="${outdir}/${obsid}/raw/${obsid}.metafits"
mkdir -p "${outdir}/${obsid}/prep"
export prep_uvfits="${outdir}/${obsid}/prep/birli_${obsid}_${timeres_s}s_${freqres_khz}kHz.uvfits"
export prep_ms="${outdir}/${obsid}/prep/birli_${obsid}_${timeres_s}s_${freqres_khz}kHz.ms"
export cotter_ms="${outdir}/${obsid}/prep/cotter_${obsid}_${timeres_s}s_${freqres_khz}kHz.ms"

mkdir -p "${outdir}/${obsid}/cal"
mkdir -p "${outdir}/${obsid}/ionoqa"
mkdir -p "${outdir}/${obsid}/img"

export MWA_BEAM_FILE="/opt/cal/mwa_full_embedded_element_pattern.h5"
export full_srclist="/opt/cal/srclist_pumav3_EoR0LoBESv2_fixedEoR1pietro+ForA_phase1+2_edit.txt"
export rts_patch_srclist="${outdir}/${obsid}/rts/srclist_pumav3_EoR0aegean_EoR1pietro+ForA_1090359520_patch1000.txt"
export rts_peel_srclist="${outdir}/${obsid}/rts/srclist_pumav3_EoR0aegean_EoR1pietro+ForA_1090359520_peel1000.txt"
# export hyp_srclist="${outdir}/${obsid}/cal/hyp_srclist_${obsid}_${dical_name}.fits"
export hyp_srclist="${outdir}/${obsid}/cal/${obsid}_reduced_n8000.yaml"
export hyp_soln="${outdir}/${obsid}/cal/hyp_soln_${obsid}_${dical_name}.fits"
export cal_uvfits="${outdir}/${obsid}/cal/hyp_${obsid}_${dical_name}_${apply_name}.uvfits"
export cal_ms="${outdir}/${obsid}/cal/hyp_${obsid}_${dical_name}_${apply_name}.ms"
export cal_ms_trunc="${outdir}/${obsid}/cal/hyp_${obsid}_${dical_name}_${apply_name}_trunc.ms"
export cal_model="${outdir}/${obsid}/cal/hyp_model_${obsid}_${dical_name}.ms"

export sub_ms="${outdir}/${obsid}/cal/hyp_sub_${obsid}_${dical_name}_${apply_name}.ms"
export residual_ms="${outdir}/${obsid}/cal/hyp_residual_${obsid}_${dical_name}_${apply_name}.ms"

export ionosub_ms="${outdir}/${obsid}/cal/hyp_ionosub_${obsid}_${dical_name}_${apply_name}.ms"
# export ionosub_ms_trunc="${outdir}/${obsid}/cal/hyp_ionosub_${obsid}_${dical_name}_${apply_name}_trunc.ms"
export ionosub_json="${outdir}/${obsid}/cal/hyp_ionosub_consts_${obsid}_${dical_name}_${apply_name}.json"
export ionosub_qa="${outdir}/${obsid}/ionoqa/hyp_ionosub_${obsid}_${dical_name}_${apply_name}.json"
# export ionosub_json_trunc="${outdir}/${obsid}/cal/hyp_ionosub_consts_${obsid}_${dical_name}_${apply_name}_trunc.json"
export ionosub_cpu_ms="${outdir}/${obsid}/cal/hyp_ionosub_cpu_${obsid}_${dical_name}_${apply_name}.ms"
export ionosub_cpu_json="${outdir}/${obsid}/cal/hyp_ionosub_cpu_consts_${obsid}_${dical_name}_${apply_name}.json"
export ionosub_cpu_qa="${outdir}/${obsid}/ionoqa/hyp_ionosub_cpu_${obsid}_${dical_name}_${apply_name}.json"

export ionosub_old_uvfits="${outdir}/${obsid}/cal-old/hyp_${obsid}_ionosub_${dical_name}_${apply_name}_${ionosub_name}.uvfits"
export ionosub_old_ms="${outdir}/${obsid}/cal-old/hyp_${obsid}_ionosub_${dical_name}_${apply_name}_${ionosub_name}.ms"
export ionosub_old_json="${outdir}/${obsid}/cal-old/hyp_peel_${obsid}_ionosub_${dical_name}_${apply_name}_${ionosub_name}_uv.json"

export ionosub_oldsim_ms="${outdir}/${obsid}/cal/hyp_ionosub_oldsim_${obsid}.ms"
export ionosub_oldsim_json="${outdir}/${obsid}/cal/hyp_ionosub_consts_oldsim_${obsid}.json"
export ionosub_oldsim_qa="${outdir}/${obsid}/ionoqa/hyp_ionosub_oldsim_${obsid}.json"

export hyp_test_peel_gpu_multi_ms=/tmp/hypertest/test_peel_gpu_multi_source.ms

# ###### #
# IMPORT #
# ###### #

# [ ! -d "$ionosub_old_ms" ] && [ -f "$ionosub_old_uvfits" ] && \
#     docker run -it \
#     -v /data:/data \
#     d3vnull0/casa:latest \
#     casa -c "importuvfits('${ionosub_old_uvfits}', '${ionosub_old_ms}')"

# ########## #
# PREPROCESS #
# ########## #
# there are two options here:

## option 1: download raw vis, manual Birli processing

# giant-squid submit-vis --wait "${obsid}" --delivery acacia
# pushd "${outdir}/${obsid}/raw"
#     giant-squid download ${obsid}
# popd
pushd "${outdir}/${obsid}/prep"
    # for uvfits
    [ -f ${prep_uvfits} ] || birli ${birli_args} \
        --avg-freq-res ${freqres_khz} --avg-time-res ${timeres_s} \
        --flag-edge-width ${edgewidth_khz} \
        -u "${prep_uvfits}" \
        -m "${metafits}" \
        ${outdir}/${obsid}/raw/${obsid}_2*.fits
    # for ms
    # birli ${birli_args} \
    #     --avg-freq-res ${freqres_khz} --avg-time-res ${timeres_s} \
    #     --flag-edge-width ${edgewidth_khz} \
    #     -M "${prep_ms}" \
    #     -m "${metafits}" \
    #     ${outdir}/${obsid}/raw/${obsid}_2*.fits
    # cotter equivalent
    # cotter \
    #     -freqres ${freqres_khz} -timeres ${timeres_s} \
    #     -edgewidth ${edgewidth_khz} \
    #     -o "${cotter_ms}" \
    #     -m "${metafits}" \
    #     -allowmissing \
    #     -noantennapruning \
    #     -noflagautos \
    #     -nostats \
    #     -norfi \
    #     ${outdir}/${obsid}/raw/${obsid}_2*.fits
popd

## option 2: use birli from ASVO

# pushd "${outdir}/${obsid}/raw"
#     wget -O "${obsid}.metafits" "http://ws.mwatelescope.org/metadata/fits?obs_id=${obsid}&include_ppds=1"
# popd
# pushd "${outdir}/${obsid}/prep"
#     giant-squid submit-conv --wait "${obsid}" \
#         --parameters timeres=${timeres_s},freqres=${freqres_khz},conversion=uvfits,processor=birli
#     giant-squid download ${obsid}
#     mv "${obsid}.uvfits" "${prep_uvfits}"
# popd

# ####### #
# PREP QA #
# ####### #

# ######### #
# CALIBRATE #
# ######### #

# model shitlist
# hyperdrive di-calibrate --uvw-min 30l -n 1 \
#     --data "${metafits}" "${cal_ms}" \
#     --source-list /data/dev/1090359520/cal/shitlist.txt \
#     --model-filenames "${cal_model}"

pushd "${outdir}/${obsid}/cal"
    [ -f "${hyp_srclist}" ] || hyperdrive srclist-by-beam \
        -m "${metafits}" \
        -n 8000 \
        "${full_srclist}" \
        "${hyp_srclist}"

    [ -f "${hyp_soln}" ] || hyperdrive di-calibrate ${=dical_args} \
        --data "${metafits}" "${prep_uvfits}" \
        --source-list "${full_srclist}" \
        --outputs "${hyp_soln}" \
        --model-filenames "${cal_model}"

    hyperdrive solutions-plot -m "${metafits}" "${hyp_soln}" --no-ref-tile

    [ -d "${cal_ms}" ] || hyperdrive solutions-apply ${=apply_args} \
        --data "${metafits}" "${prep_uvfits}" \
        --solutions "${hyp_soln}" \
        --outputs "${cal_ms}"
        # "${cal_uvfits}"
        #

    # // truncated ms
#     casa -c $(cat <<EoF
#     print('hi')
#     tb.open('$cal_ms')
#     times=tb.taql('SELECT UNIQUE TIME FROM ' + tb.name()).getcol('TIME')
#     mid_idx = len(times)/2
#     tb.query('TIME >= ' + str(times[mid_idx]) + ' AND TIME < ' + str(times[mid_idx+1])).copy('$cal_ms_trunc', deep=True)
#     tb.close()
#     print('bye')
# EoF
# )

# ######## #
# SUBTRACT #
# ######## #
    [ -d $sub_ms ] || hyperdrive vis-subtract \
        --data "${cal_ms}" \
        --source-list "${full_srclist}" \
        --outputs "${sub_ms}" \
        --invert \
        --num-sources 1000

# #### #
# PEEL #
# #### #

    # just residual
    [ -d $residual_ms ] || hyperdrive peel \
        --data "${cal_ms}" "${metafits}" \
        --source-list "${full_srclist}" \
        --outputs "${residual_ms}" \
        --sub 1000 --iono-sub 0 \
        2>&1 | tee residual.log

    [ -d $ionosub_ms ] || hyperdrive peel -v \
        --data "${cal_ms}" "${metafits}" \
        --source-list "${full_srclist}" \
        --outputs "${ionosub_ms}" "${ionosub_json}" \
        --sub 8000 ${=ionosub_args} \
        2>&1 | tee peel.log
    # our best peel attempt
        # --sub 1000 --iono-sub 1000 \
        # --iono-freq-average-factor 1 \
        # --low-res-iono-freq-average 1 \
        # --short-baseline-sigma 100 \
        # --uvw-min 50l \
        # --convergence 0.5 \
    # debugging
    # export RUST_BACKTRACE=full
    # cargo build --release --features=cuda,gpu-single \
    # && target/release/hyperdrive peel -vvv --no-progress-bars \
    #     --data "${cal_ms}" "${metafits}" \
    #     --source-list "${hyp_srclist}" \
    #     --outputs "${ionosub_ms}" "${ionosub_json}" \
    #     --sub 8000 \
    #     --num-passes 2 \
    #     --num-loops 2 \
    #     --iono-sub 10 2>&1 | tee peel-debug.log
        # iono-freq-average-factor 1



    # hyperdrive peel \
    #     --data "${cal_ms_trunc}" \
    #     --beam-file "${MWA_BEAM_FILE}" \
    #     --source-list "${hyp_srclist}" \
    #     --outputs "${ionosub_ms_trunc}" "${ionosub_json_trunc}" \
    #     --sub 8000 \
    #     --iono-sub 1000
    # cargo build --no-default-features --features=hdf5-static
    # target/debug/hyperdrive peel -vvv --no-progress-bars \
    #     --data "${cal_ms}" "${metafits}" \
    #     --source-list "${hyp_srclist}" \
    #     --outputs "${ionosub_cpu_ms}" "${ionosub_cpu_json}" \
    #     --sub 8000 \
    #     --num-passes 2 \
    #     --num-loops 2 \
    #     --iono-sub 10 2>&1 | tee peel-cpu.log
    # simulate hyperdrive old
    # hyperdrive-old peel --source-dist-cutoff=90 --veto-threshold 0.005 --data 1090359520.metafits hyp_1090359520_30l_src4k_8s_80kHz.uvfits --beam-file /pawsey/mwa/mwa_full_embedded_element_pattern.h5 --source-list srclist_pumav3_EoR0LoBESv2_fixedEoR1pietro+ForA_phase1+2_edit.txt --iono-sub 1000 --sub 8000 --outputs hyp_1090359520_ionosub_30l_src4k_8s_80kHz_i1000.uvfits hyp_peel_1090359520_ionosub_30l_src4k_8s_80kHz_i1000_uv.json
    # [ -f $ionosub_oldsim_ms ] || hyperdrive peel \
    #     --source-dist-cutoff=90 --veto-threshold 0.005 \
    #     --data "${cal_ms}" "${metafits}" \
    #     --source-list "${full_srclist}" \
    #     --iono-sub 1000 --sub 8000 \
    #     --short-baseline-sigma 0.0001 \
    #     --convergence 1.0 \
    #     --outputs "$ionosub_oldsim_ms" "$ionosub_oldsim_json"
popd

# ###### #
# IONOQA #
# ###### #

# 2s timestamps:
# 0  1090359521
# 32 1090359585
# 33 1090359587
# 34 1090359589
# 35 1090359591
# avg 1090359588

docker run -it \
    -v ~/src/MWAEoR-Pipeline/templates/:/templates \
    -v /data:/data \
    --entrypoint /usr/bin/python \
    d3vnull0/mwa_qa:latest \
    /templates/cthulhuplot.py \
    --offsets "${ionosub_json}" \
    --obs-frequency 182000000 \
    --obs-radec 0 -27 \
    --obsid 1090359588 \
    --time-res 8 \
    --show-axes \
    --json "${ionosub_qa%%.json}.json" \
    --csv "${ionosub_qa%%.json}.csv" \
    --tec "${ionosub_qa%%.json}_tec.png" \
    --plot "${ionosub_qa%%.json}.png"
docker run -it \
    -v ~/src/MWAEoR-Pipeline/templates/:/templates \
    -v /data:/data \
    --entrypoint /usr/bin/python \
    d3vnull0/mwa_qa:latest \
    /templates/cthulhuplot.py \
    --offsets "${ionosub_oldsim_json}" \
    --obs-frequency 182000000 --obs-radec 0 -27 \
    --json "${ionosub_oldsim_qa%%.json}.json" \
    --csv "${ionosub_oldsim_qa%%.json}.csv" \
    --plot "${ionosub_oldsim_qa%%.json}.png" \
    --tec "${ionosub_oldsim_qa%%.json}_tec.png"

# unzip /data/dev/rts_offsets/rts.zip ${obsid}.yaml && cp ${obsid}.yaml ${outdir}/${obsid}/rts/
export rts_offsets="${outdir}/${obsid}/rts/${obsid}.yaml"
export rts_tec="${outdir}/${obsid}/rts/tec_${obsid}.png"
export rts_qa="${outdir}/${obsid}/rts/qa_${obsid}.png"
docker run -it \
    -v ~/src/MWAEoR-Pipeline/templates/:/templates \
    -v /data:/data \
    --entrypoint /usr/bin/python \
    d3vnull0/mwa_qa:latest \
    /templates/cthulhuplot.py \
    --offsets ${rts_offsets} \
    --obs-frequency 182000000 --obs-radec 0 -27 \
    --timestep 8 \
    --json "${rts_offsets%%.yaml}_qa.json" \
    --csv "${rts_offsets%%.yaml}.csv" \
    --plot "${rts_qa}" \
    --tec "${rts_tec}"

# ##### #
# IMAGE #
# ##### #

export img_base=""
# export img_base="-parallel-gridding 4"
# export img_base="-parallel-gridding 24 -interval 4 5"
# export img_base="-parallel-gridding 24 -interval 3 7"
# export img_base="-gridder idg -idg-mode hybrid"
# export img_base="-parallel-gridding 4 -intervals-out 14"

unset imgs
if [[ -n "$ZSH_VERSION" ]]; then
    typeset -A imgs
else
    declare -A imgs
fi
# imgs["fast"]="$img_base -weight briggs -1.0 -size 2048 2048 -scale 40asec -niter 0"
# imgs["primary"]="$img_base -weight briggs -1.0 -size 2048 2048 -scale 1amin -niter 100 -channels-out 4 -join-channels"
# imgs[points]="$img_base -weight uniform -size 4096 4096 -scale 40asec -niter 0 -channels-out 4 -join-channels"
imgs[uni]="$img_base -weight uniform -size 4096 4096 -scale 40asec -niter 0"
# imgs[uni_24ch]="$img_base -weight uniform -size 4096 4096 -scale 40asec -niter 1000 -channels-out 24 -join-channels"
# imgs[nat]="$img_base -weight natural -size 4096 4096 -scale 40asec -niter 0 -channels-out 4 -join-channels"
# imgs[nat100]="$img_base -weight natural -minuv-l 100 -size 4096 4096 -scale 40asec -niter 0 -channels-out 4 -join-channels"
# imgs["allsky-wb0-4096x105as-24ch-tg10am-nodc"]="$img_base -weight briggs +0.5 -size 4096 4096 -scale 105asec -channels-out 24 -join-channels -taper-gaussian 10amin -niter 0"
# imgs["allsky-wb+.5-1024x420as-96ch-nodc"]="$img_base -weight briggs +0.5 -size 1024 1024 -scale 420asec -channels-out 96 -join-channels -niter 0"
# imgs["allsky-wb+0-1024x420as-96ch-nodc"]="$img_base -weight briggs +0.0 -size 1024 1024 -scale 420asec -channels-out 96 -join-channels -niter 0"
# imgs["allsky-wb-.5-1024x420as-96ch-nodc"]="$img_base -weight briggs -0.5 -size 1024 1024 -scale 420asec -channels-out 96 -join-channels -niter 0"
# imgs["allsky-wb+.5-1024x420as-96ch-tg10am-nodc"]="$img_base -weight briggs +0.5 -size 1024 1024 -scale 420asec -channels-out 96 -join-channels -taper-gaussian 10amin -niter 0"
# imgs["allsky-wb+0-1024x420as-96ch-tg10am-nodc"]="$img_base -weight briggs +0.0 -size 1024 1024 -scale 420asec -channels-out 96 -join-channels -taper-gaussian 10amin -niter 0"
# imgs["allsky-wb-.5-1024x420as-96ch-tg10am-nodc"]="$img_base -weight briggs -0.5 -size 1024 1024 -scale 420asec -channels-out 96 -join-channels -taper-gaussian 10amin -niter 0"

# export img_name="fast"
# export img_args="$img_base -weight briggs -1.0 -size 2048 2048 -scale 40asec -niter 0"

# export img_name="primary"
# export img_args="$img_base -weight briggs -1.0 -size 2048 2048 -scale 1amin -niter 100 -parallel-deconvolution 512 -channels-out 96 -join-channels"

# export img_name="allsky-wb+.5-4096x90as-96ch-tg10am-nodc"
# export img_args="$img_base -weight briggs +0.5 -size 4096 4096 -scale 90asec -channels-out 96 -join-channels -taper-gaussian 10amin -niter 0"

# export img_name="allsky-wb+.5-1024x420as-24ch-tg10am-nodc"
# export img_args="$img_base -weight briggs +0.5 -size 1024 1024 -scale 420asec -channels-out 24 -join-channels -taper-gaussian 10amin -niter 0"

# export img_name="allsky-wb+0-1024x420asec-24ch-tg10am-nodc"
# export img_args="$img_base -weight briggs +0.0 -size 1024 1024 -scale 420asec -channels-out 24 -join-channels -taper-gaussian 10amin -niter 0"

# export img_name="allsky-wb-.5-1024x420asec-24ch-tg10am-nodc"
# export img_args="$img_base -weight briggs -0.5 -size 1024 1024 -scale 420asec -channels-out 24 -join-channels -taper-gaussian 10amin -niter 0"

# for img_name in ("${!imgs[@]}"); do
    # export img_args="${imgs[$img_name]}"

for img_name img_args in ${(kv)imgs}; do
    mkdir -p "${outdir}/${obsid}/img-${img_name}"
    pushd "${outdir}/${obsid}/img-${img_name}"
        wsclean ${=img_args} \
            -name wsclean_hyp_${obsid}_${dical_name}_${apply_name}_${img_name} \
            "${cal_ms}"
        wsclean ${=img_args} \
            -name wsclean_hyp_sub_${obsid}_${dical_name}_${apply_name}_${img_name} \
            "${sub_ms}"
        wsclean ${=img_args} \
            -name wsclean_hyp_residual_${obsid}_${dical_name}_${apply_name}_${img_name} \
            "${residual_ms}"
        time wsclean ${=img_args} \
            -name wsclean_hyp_ionosub_${obsid}_${dical_name}_${apply_name}_${img_name} \
            "${ionosub_ms}"

        # wsclean ${=img_args} \
        #     -name wsclean_hyp_ionosub_oldsim_${obsid}_${dical_name}_${apply_name}_${img_name} \
        #     "${ionosub_oldsim_ms}"
        # wsclean ${=img_args} \
        #     -name wsclean_hyp_ionosub_old_${obsid}_${dical_name}_${apply_name}_${img_name} \
        #     "${ionosub_old_ms}"
        # wsclean ${=img_args} \
        #     -name wsclean_hyp_ionosub_cpu_${obsid}_${dical_name}_${apply_name}_${img_name} \
        #     "${ionosub_cpu_ms}"
        wsclean ${=img_args} \
            -name wsclean_hyp_model_${obsid}_${dical_name}_${apply_name}_${img_name} \
            "${cal_model}"
    popd
done

# make thumbnails
mkdir -p "${outdir}/${obsid}/img_qa"
for img_name img_args in ${(kv)imgs}; do
    for fits in $(cd ${outdir}/${obsid}/img-${img_name}; ls *-{psf,image}.fits); do
        docker run -it -v /home/dev/src/MWAEoR-Pipeline/templates:/templates -v /data:/data --entrypoint python d3vnull0/mwa_qa:latest \
            /templates/thumbnail.py \
            --fits="${outdir}/${obsid}/img-${img_name}/${fits}" \
            --thumb="${outdir}/${obsid}/img_qa/${fits}.png" \
            --title="${obsid} ${dical_name} ${apply_name}"$'\n'"${img_name} MFS"
    done
done

exit 0

# [2023-10-11T15:05:51Z DEBUG] peel loop finished: p0 SNG002646-3655   @ r +0.12 d -0.64 |o a+0.00e0 b+0.00e0 g1.00 |n a+2.81e-4 b+5.04e-4 g-0.14

# #### #
# GIFS #
# #### #

ffmpeg -y -framerate 5 -pattern_type glob \
    -i ${ionosub_plot%%.png}'-t*.png' \
  -lavfi 'palettegen=stats_mode=single[pal],[0:v][pal]paletteuse=new=1' "${ionosub_plot%%.png}".gif



# ##### #
# TESTS #
# ##### #

export test=test_peel_cpu_multi_source
cargo test --no-default-features --features=hdf5-static --release -- $test --nocapture 2>&1 \
 | tee $test.log;


mkdir -p "${outdir}/${obsid}/img-test"
export test=test_peel_gpu_multi_source
cargo test --features=cuda --release -- $test --nocapture 2>&1 \
 | tee $test.log;
pushd "${outdir}/${obsid}/img-test" && \
wsclean -name $test \
    -size 4096 4096 -scale 32asec \
    -weight uniform -niter 0 \
    /tmp/hypertest/$test.ms && \
[ -d "/tmp/hypertest/${test}_prec.ms" ] && wsclean -name ${test}_prec \
    -size 4096 4096 -scale 32asec \
    -weight uniform -niter 0 \
    /tmp/hypertest/${test}_prec.ms
popd

# #####
# DEBUGGING MITCH
# #####

cargo install --path . --features=cuda,gpu-single \
&& hyperdrive peel -v \
        --data "${cal_ms}" "${metafits}" \
        --source-list "${rts_peel_srclist}" \
        --outputs "${ionosub_ms}" "${ionosub_json}" \
        --sub 1000 --iono-sub 3 \
        2>&1 | tee peel.log
pushd "${outdir}/${obsid}/img-uni"
export img_name="uni"
export img_args="$img_base -weight uniform -size 4096 4096 -scale 40asec -niter 0"
wsclean ${=img_args} \
    -name wsclean_hyp_ionosub_${obsid}_${dical_name}_${apply_name}_${img_name} \
    "${ionosub_ms}"
popd