tpu wav2vec2.0 command

gistfile1.txt

python \
 $p/fairseq/train.py \
 $HOME/data/w2v/manifest \
--tpu \
--distributed-world-size 8 \
--max-update 500000 \
--max-epoch 5 \
--num-batch-buckets 1 \
 --required-batch-size-multiple 4 \
 --max-sentences 4 \
--enable-padding \
--log-interval 200 \
--encoder-layers 12 \
--num-workers 6 \
--task audio_pretraining \
--criterion wav2vec \
--arch wav2vec2 \
--quantize-targets \
--extractor-mode default \
--conv-feature-layers '[(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512,2,2)] * 2' \
--final-dim 256 \
--latent-vars 320 \
--latent-groups 2 \
--latent-temp '(2,0.5,0.999995)' \
--infonce \
--optimizer adam \
--adam-betas '(0.9,0.98)' \
--adam-eps 1e-06 \
--lr-scheduler polynomial_decay \
--lr 0.0005 \
--warmup-updates 32000 \
--mask-length 10 \
--mask-prob 0.65 \
--mask-selection static \
--mask-other 0 \
--mask-channel-prob 0.65 \
--encoder-layerdrop $ENCLAYERDROP \
--dropout-input 0.1 \
--dropout-features 0.1 \
--feature-grad-mult 0.1 \
--loss-weights '[0.1, 10]' \
--conv-pos 128 \
--conv-pos-groups 16 \
--num-negatives 100 \
--cross-sample-negatives 0 \
--max-sample-size 250000 \
--min-sample-size 32000 \
--no-save \
--dropout 0.1 \
--attention-dropout 0.1 \
--weight-decay 0.01 \
--max-tokens 1400000 \
--skip-invalid-size-inputs-valid-test \
--log-format simple \
--disable-validation \
--seed 1234 \
--ddp-backend no_c10d