run_on_cluster.bash

#! /bin/bash
# @nolint


. crun_tools.bash



basedir=/mnt/vol/gfsai-flash3-east/ai-group/users/matthijs/bench_standalone_codec

lars=$basedir/lars
logdir=$basedir/logs


mkdir -p $lars $logdir
chmod 777 $logdir

compile faiss/benchs/bench_all_ivf bench_standalone_codec
if false; then


for db in bigann1M sift1M deep1M sentences_de sentences_ru low_shot_resnet50; do
# for db in sentences_ru; do
    for key in Pad256,ITQ256,LSH Pad128,ITQ128,LSH ITQ64,LSH RR256,LSHt RR128,LSHt; do

        run_on_1machine bin.$db.$key.d \
                        $exe --db $db --factorykey $key --eval_search_hamming
    done

done



for db in bigann1M sift1M deep1M sentences_de sentences_ru low_shot_resnet50; do

#            SQfp16 SQ8 SQ6 SQ4 \
#               PCAR256,SQ8 PCAR512,SQ8 \
#               PCAR128,SQ8 PCAR64,SQ8 \
#               PCAR128,SQ4 PCAR64,SQ4 \
#               OPQ64_256,PQ64 OPQ32_128,PQ32 OPQ16_64,PQ16 \
#               OPQ128_512,PQ128


    for key in \
        OPQ128_512,PQ128x10 PCAR512,PQ128x10 \
        OPQ128_512,PQ128x12 OPQ128_512,PQ128x14 \
        OPQ16_64,PQ16x10 OPQ16_64,PQ16x12 OPQ16_64,PQ16x14 \
        OPQ24_96,PQ24 OPQ24_96,PQ12 \
        OPQ8_48,PQ8 OPQ8_48,PQ8x10 OPQ8_48,PQ8x12
    do
        run_on_1machine float.$db.$key.b \
                        $exe --db $db --factorykey $key \
                        --eval_search --eval_reconstruction

    done

done


# .b: redo all IVF / IMI experiments due to bug in decoding of centroid
# .c: use residuals

for db in bigann1M sift1M deep1M sentences_de sentences_ru low_shot_resnet50; do
#        OPQ32_128,PQ32x10 OPQ32_128,PQ32x12 \
#        OPQ64_256,PQ64x10 OPQ64_256,PQ64x12 \
#        OPQ32_128,IVF16384,PQ32 \
#        OPQ32_128,IMI2x10,PQ32 \
#        OPQ32_128,IMI2x12,PQ32 \
#        OPQ32_128,IMI2x14,PQ32

    for key in \
        OPQ32_128,Residual2x14,PQ32x12 \
        OPQ32_128,Residual2x14,PQ32x14
    do
        run_on_1machine float2.$db.$key.e \
                        $exe --db $db --factorykey $key \
                        --eval_search --eval_reconstruction \
                        --maxtrain 1000000
    done

done


for db in sentences_de low_shot_resnet50; do
    for key in \
        PCAR1024,PQ256 PCAR1024,PQ256x12 \
        PCAR1024,Residual2x14,PQ256 PCAR1024,Residual2x14,PQ256x12 \
        PCAR1024,PQ512 PCAR1024,PQ512x12
    do
        run_on_1machine expensive.$db.$key.e \
                        $exe --db $db --factorykey $key \
                        --eval_search --eval_reconstruction \
                        --maxtrain 1000000
    done

done


# lattice experiments!

# study on 32 byte target, 4x64 bit
for db in sentences_de low_shot_resnet50; do

    for norm_nbit in 8; do

        case $norm_nbit in
            4) dim_r2s='20,118 24,61 28,40 32,30 36,24 40,21 44,18 48,17' ;;
            6) dim_r2s='20,101 24,54 28,36 32,27 36,22 40,19 44,17 48,16' ;;
            8) dim_r2s='20,86 24,47 28,32 32,25 36,20 40,18 44,16 48,15' ;;
        esac


        for dim_r2 in $dim_r2s; do
            dim=${dim_r2%,*}
            r2=${dim_r2#*,}
            key=PCAR$((4 * dim)),ZnLattice4x${r2}_${norm_nbit}

            run_on_1machine target32byte.$db.$key.b \
                            $exe --db $db --factorykey $key \
                            --eval_search --eval_reconstruction

        done
    done
done

fi


for db in sift1M deep1M sentences_de low_shot_resnet50; do

    for nsq in 1 2 4 8 16 32 64; do

        for norm_nbit in 4 6; do

            case $norm_nbit in
                4) dim_r2s='28,40 24,61 ' ;;
                6) dim_r2s='24,54 ' ;;
                8) dim_r2s='24,47 ' ;;
            esac

            for dim_r2 in $dim_r2s; do
                dim=${dim_r2%,*}
                r2=${dim_r2#*,}
                key=PCAR$((nsq * dim)),ZnLattice${nsq}x${r2}_${norm_nbit}

                run_on_1machine latsweep.$db.$key.a \
                            $exe --db $db --factorykey $key \
                            --eval_search --eval_reconstruction
            done

        done
    done
done