Building Vocabulary

Vocab

def build_datasets_vocab(root_file, captions_file, transform, split=0.15):
    df = pd.read_csv(captions_file)
    
    vocab = {}
    def create_vocab(caption):
        tokens = [token.lower() for token in word_tokenize(caption)]
        for token in tokens: 
            if token not in vocab: 
                vocab[token] = len(vocab)
            
    df["caption"].apply(create_vocab)
    
    train, valid = train_test_split(df, test_size=split, random_state=42)
    return My_Flickr1k(root_file, train.values, transform), \
           My_Flickr1k(root_file, valid.values, transform), \
           vocab