reaching_for_sota.sh

#!/bin/bash
set -e
set -x

export TASK=SQUAD
export SQUAD_VERSION=2.0
export MODEL_NAME=large
export SQUAD_DATA=/home/ubuntu/SQuAD_data

export BS=2
export ACCUMULATE=16
GBS=$(($BS * $ACCUMULATE))
export LR=3e-5
export MSL=512
export LWD=0.9
export WD=0.01
export EP=3
export MGN=0.1
export SEED=28
export WUR=0.1
export OUTPUT_DIR=fucking_sota/${TASK}${SQUAD_VERSION}_${MODEL_NAME}_${GBS}_${LR}_${WD}_${EP}_${MGN}_${LWD}_${SEED}
pip3 install numpy
set +x

mpirun -np 4 -H localhost:4 -bind-to none -map-by slot python3 -m run_squad \
    --model_name=google_electra_${MODEL_NAME} \
    --do_eval \
    --do_train \
    --data_dir=${SQUAD_DATA} \
    --save_interval 200 \
    --output_dir=${OUTPUT_DIR} \
    --layerwise_decay=${LWD} \
    --max_saved_ckpt=10 \
    --all_evaluate \
    --gpus=0,1,2,3 \
    --num_accumulate=${ACCUMULATE} \
    --version=${SQUAD_VERSION} \
    --batch_size=${BS} \
    --lr=${LR} \
    --wd=${WD} \
    --seed=${SEED} \
    --max_seq_length=${MSL} \
    --eval_batch_size=8 \
    --epochs=${EP} \
    --warmup_ratio=${WUR} \
    --overwrite_cache \
    --max_grad_norm=${MGN} \
    --comm_backend horovod \