tensor2tensor_imdb_sentiment_analysis_subwords_multigpu.sh

PROBLEM=sentiment_imdb_characters
MODEL=transformer_encoder
HPARAMS=transformer_base
DATA_DIR=$HOME/t2t_data
TMP_DIR=/tmp/t2t_datagen
TRAIN_DIR=$HOME/t2t_train/$PROBLEM/$MODEL-$HPARAMS
BATCH_SIZE=2048
WORKER_GPU=2
TRAIN_STEPS=500000

mkdir -p $DATA_DIR $TMP_DIR

tensorboard --logdir $TRAIN_DIR &

t2t-datagen \
  --data_dir=$DATA_DIR \
  --tmp_dir=$TMP_DIR \
  --problem=$PROBLEM

t2t-trainer \
  --data_dir=$DATA_DIR \
  --problem=$PROBLEM \
  --model=$MODEL \
  --hparams_set=$HPARAMS \
  --output_dir=$TRAIN_DIR \
  --tmp_dir=$TMP_DIR \
  --batch_size=$BATCH_SIZE \
  --worker_gpu=$WORKER_GPU \
  --train_steps=$TRAIN_STEPS \
  --hparams=eval_drop_long_sequences=True

zzzzz

https://github.com/tensorflow/tensor2tensor/issues/266

--local_eval_frequency=0

@martinpopel I solved it by upgrading to TF 1.5.0 (cuda 9.0, cuDNN 7.0). Before that I had TF 1.4.1. Now the regular continuous_train_and_eval process with --worker_gpu=8 works out of the box.

charlevel MultiLabelModality ?