calibrate a picket fence observation with and without signal chain corrections

cal_picket_sig.sh

#!/bin/bash
set -eux

export obsid=1369821496
export outdir="/cygnus/dev/${obsid}"

export hyperdrive_devel="cargo +nightly -Z unstable-options -C ~/src/hyperdrive-devel run --release --quiet --features=hdf5-static,cuda,gpu-single --"
export hyperdrive_main="hyperdrive"
export birli="cargo +nightly -Z unstable-options -C ~/src/Birli run --release --quiet --"
# export srclist="/opt/cal/srclist_pumav3_EoR0LoBES_EoR1pietro_CenA-GP_2023-11-07.yaml"
# export model_name="_jlobes202311"
export srclist="/opt/cal/GGSM_updated.fits"
export model_name="_ggsm"

# birli settings
export timeres_s=2
export freqres_khz=40
export edgewidth_khz=160
# without signal chain corrections
# export birli_args=""
# export prep_name="_${timeres_s}s_${freqres_khz}kHz_nosig"
# with signal chain corrections
export birli_args="--signal-chain"
export prep_name="_${timeres_s}s_${freqres_khz}kHz_sig"

# Srclist settings
export num_sources=4000   # number of sources to model in calibration
export dist_cutoff=90     # distance cutoff for srclist in degrees
export veto_thresh=0.005  # minimum source brightness in Jy
export srclist_args="-n ${num_sources} --source-dist-cutoff=${dist_cutoff} --veto-threshold ${veto_thresh}"
export srclist_name="_${num_sources}src_${dist_cutoff}deg_${veto_thresh}Jy"

# Calibration settings
export uvw_min="50l"       # minimum uvw cutoff (lamda)
export max_iters=300       # number of di-cal iterations
export stop_thresh=1e-20   # stop threshold for di-cal
export dical_freq="80kHz"  # frequency resolution for di-cal
export dical_args="--uvw-min ${uvw_min} --max-iterations ${max_iters} --stop-thresh ${stop_thresh} --freq-average ${dical_freq}"
export dical_name="_${uvw_min}_${num_sources}src_${max_iters}it_${dical_freq}"

mkdir -p $outdir/log
mkdir -p $outdir/raw
export raw=${outdir}/raw/${obsid}_2*.fits
export metafits=${outdir}/raw/${obsid}.metafits
export metajson="${metafits%%.fits}.json"
export tiledata="${metafits%%.fits}_tiles.tsv"
export rxtiles="${metafits%%.fits}_rxtiles.txt"

mkdir -p $outdir/prep
export prep_uvfits="${outdir}/prep/birli_${obsid}${prep_name}.uvfits"
export prep_uvfits_glob=${outdir}/prep/birli_${obsid}${prep_name}_ch*.uvfits

mkdir -p $outdir/cal${model_name}
export hyp_soln_glob="${outdir}/cal${model_name}/hyp_soln_${obsid}${prep_name}_ch*.fits"
mkdir -p $outdir/calqa${model_name}

# ######## #
# METADATA #
# ######## #

ls -1 $raw || (cd $outdir/raw; echo giant-squid download $obsid)

[ -f "$metafits" ] || wget -O "$metafits" $'http://ws.mwatelescope.org/metadata/fits?obs_id='${obsid}

[ -f "$tiledata" ] || docker run -it \
    -v ~/src/MWAEoR-Pipeline/templates/:/templates \
    -v /cygnus:/cygnus \
    --entrypoint /usr/bin/python \
    d3vnull0/mwa_qa:latest \
    /templates/metajson.py \
    --metafits "${metafits}" \
    --json "${metajson}" \
    --tsv "${tiledata}"

# #### #
# PREP #
# #### #
ls -1 $prep_uvfits_glob || eval RUST_LOG=birli=trace $birli ${birli_args} \
    --avg-freq-res ${freqres_khz} --avg-time-res ${timeres_s} \
    --flag-edge-width ${edgewidth_khz} \
    -u "${prep_uvfits}" \
    -m "${metafits}" \
    $raw \
    2>&1 | tee ${outdir}/log/birli_${obsid}${prep_name}.log

# ### #
# CAL #
# ### #

for prep_uvfits in $(ls -1 $prep_uvfits_glob); do
    export prep_base="${prep_uvfits##*/birli_}"
    export prep_base=${prep_base%.uvfits}
    [ -d "${outdir}/cal${model_name}/" ] || mkdir -p "${outdir}/cal${model_name}/"
    (
        cd "${outdir}/cal${model_name}/"
        export hyp_srclist="${PWD}/srclist_${prep_base}.yaml"
        export hyp_soln="${PWD}/hyp_soln_${prep_base}.fits"
        export cal_ms="${PWD}/hyp_${prep_base}.ms"
        export model_ms="${PWD}/model_${prep_base}.ms"
        [ -f $hyp_srclist ] || eval $hyperdrive_main srclist-by-beam ${srclist_args} \
            --metafits "${metafits}" \
            "${srclist}" -- "${hyp_srclist}"
        [ -f $hyp_soln ] || eval $hyperdrive_devel di-calibrate ${srclist_args} ${dical_args}  \
            --data "${metafits}" "${prep_uvfits}" \
            --source-list "${hyp_srclist}" \
            --outputs "${hyp_soln}"
            # --model-filenames $model_ms
        ls ${hyp_soln%.fits}*.png || eval $hyperdrive_main plot-solutions \
            --no-ref-tile \
            --max-amp=2 \
            --metafits "${metafits}" \
            "${hyp_soln}"
    )&
    sleep 5;
done

wait $(jobs -rp)

pushd "${outdir}/calqa${model_name}/"
    eval docker run -it --rm -v ${outdir}:${outdir} --entrypoint python -w "${outdir}/calqa${model_name}/" d3vnull0/mwax_mover:latest \
        /app/scripts/cal_analysis.py \
        --name "picket${prep_name}${model_name}" \
        --metafits $metafits --solns $hyp_soln_glob \
        --phase-diff-path=None
popd

# rm -rf $outdir/prep $outdir/cal${model_name} "${outdir}/${obsid} "*.{png,tsv}