bartgras

#First I've tried this, but it failed with OOM
mkdir -p ./data/v1_bigtest
mkdir -p ./train/v1_bigtest

t2t-trainer \
  --t2t_usr_dir=./data_generators \
  --generate_data \  
  --tmp_dir=/e/challenger_nmt/t2t_temp_dir \
  --data_dir=./data/v1_bigtest \
  --problems=challenger_enzh_v1 \

bartgras

import re, collections 

def get_stats(vocab): 
    pairs = collections.defaultdict(int) 
    for word, freq in vocab.items(): 
        symbols = word.split() 
        for i in range(len(symbols)-1): 
            pairs[symbols[i],symbols[i+1]] += freq
    print(pairs)
    return pairs