An example of finetuning a language model using the Huggingface Transformer library

BertFinetuning

mkdir -p output
# Install Transformers
pip install transformers

# Download the wikitext raw data files
if [ ! -d "wikitext-2-raw" ]; 
  then
    wget https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip
    unzip wikitext-2-raw-v1.zip
fi 

# Download the huggingface language model finetuning file
if [ ! -f "run_lm_finetuning.py" ]; 
  then
    wget https://raw.githubusercontent.com/huggingface/transformers/master/examples/run_lm_finetuning.py
fi

# Run the finetuning
python run_lm_finetuning.py --output_dir=output --overwrite_output_dir \
       --mlm --model_type=bert --model_name_or_path=bert-base-cased --do_train --train_data_file=wikitext-2-raw/wiki.train.raw  \
       --do_eval --eval_data_file=wikitext-2-raw/wiki.test.raw