OpenNIR script and results for #22

cedr_config.diff

(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ git diff config/cedr/_dir 
diff --git a/config/cedr/_dir b/config/cedr/_dir
index 88dab39..dda874e 100644
--- a/config/cedr/_dir
+++ b/config/cedr/_dir
@@ -1,6 +1,12 @@
 vocab=bert
 vocab.train=True
-valid_pred.batch_size=16
+
+#valid_pred.batch_size=16
+valid_pred.batch_size=2
+vocab.bert_base=bert-base-uncased
+
 test_pred.batch_size=16
 trainer.encoder_lr=2e-5
-trainer.grad_acc_batch=2
+
+#trainer.grad_acc_batch=2
+trainer.grad_acc_batch=1

results-f1.sh

# identify f1 with best performance on validation set
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ ls data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS1_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf1_bm25.100/robust_f1_bm25.100/runs/ 
11.run

# RUN=f1 test run
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ export RUN=data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS1_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf1_bm25.100/robust_f1_bm25.100/runs/11.run

# gdeval without -c (because only f1 qids are present)
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ gdeval.pl -k 20  data/datasets/robust/all.qrels $RUN|tail -n 1 
run,amean,0.43717,0.11872

# trec_eval without -c (because only f1 qids are present)
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ trec_eval -m P data/datasets/robust/all.qrels $RUN 
P_5                     all     0.5600
P_10                    all     0.4540
P_15                    all     0.4107
P_20                    all     0.3750
P_30                    all     0.3313
P_100                   all     0.1768
P_200                   all     0.0884
P_500                   all     0.0354
P_1000                  all     0.0177

results.sh

# My understanding is that the validation predictions are in robust_vaf5_bm25.100 whereas the test predictions are in robust_f5_bm25.100
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ ls data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS?_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf?_bm25.100/robust_f?_bm25.100/runs/*.run             
data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS1_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf1_bm25.100/robust_f1_bm25.100/runs/11.run
data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS2_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf2_bm25.100/robust_f2_bm25.100/runs/12.run
data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS3_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf3_bm25.100/robust_f3_bm25.100/runs/44.run
data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS4_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf4_bm25.100/robust_f4_bm25.100/runs/6.run
data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS5_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf5_bm25.100/robust_f5_bm25.100/runs/0.run

# combine the test results into a single file:
cat data/models/NRS/cedr_knrm_20q_2000d/bert_bert-base-uncased_NRS?_tr/pairwise_32x16x1_adam-0.001_encoderlr-2e-05_softmax_pos-intersect-query_neg-run/robust_trf?_bm25.100/robust_f?_bm25.100/runs/*.run > allNRS.run

# P_20 from trec_eval
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ trec_eval -c -m P data/datasets/robust/all.qrels allNRS.run
P_5                     all     0.5871
P_10                    all     0.5068
P_15                    all     0.4557
P_20                    all     0.4167
P_30                    all     0.3604
P_100                   all     0.1839
P_200                   all     0.0919
P_500                   all     0.0368
P_1000                  all     0.0184

# nDCG@20 from gdeval
(miniconda3-4.5.4) ayates@wks-15-81 OpenNIR-cedr $ gdeval.pl -c -k 20  data/datasets/robust/all.qrels allNRS.run|tail -n 1 
run,amean,0.48260,0.13216

run.sh

#!/bin/zsh
set -e

# rerunning with "no run score"
EXPID="NRS"

MODELSPACE=modelspace=$EXPID
BERT_MODEL_PARAMS="trainer.grad_acc_batch=1 valid_pred.batch_size=4 test_pred.batch_size=4"
DATACONFIG=data_dir=/home/ayates/OpenNIR-cedr/data

#for FIDX in 3 4 5; do
#for FIDX in 1 2; do
for FIDX in 1 2 3 4 5; do

FOLDCONFIG=config/robust/fold${FIDX}
    
    
python -m onir.bin.init_dataset $MODELSPACE $FOLDCONFIG $DATACONFIG

python -m onir.bin.pipeline $MODELSPACE $FOLDCONFIG $DATACONFIG config/vanilla_bert ${=BERT_MODEL_PARAMS}

python -m onir.bin.extract_bert_weights $MODELSPACE pipeline.overwrite=True pipeline.bert_weights=${EXPID}${FIDX} $FOLDCONFIG $DATACONFIG pipeline.test=true config/vanilla_bert ${=BERT_MODEL_PARAMS}

ranker=config/cedr/knrm
python -m onir.bin.pipeline $MODELSPACE $FOLDCONFIG $DATACONFIG $ranker ${=BERT_MODEL_PARAMS} vocab.bert_weights=${EXPID}${FIDX} pipeline.test=true

done