quora_data_augmentation

data_augmentation.py

nearest_syns = NearestNeighbors(n_neighbors=total_syns+1).fit(embeddings_matrix);
neighbours_mat = nearest_syns.kneighbors(embeddings_matrix[1:top_k])[1];
synonyms = {x[0]: x[1:] for x in neighbours_mat};

def augment_sentence(encoded_sentence, prob = 0.5):
    for posit in range(len(encoded_sentence)):
        if random.random() > prob:
            try:
                syns = synonyms[encoded_sentence[posit]];
                rand_syn = np.random.choice(syns);
                encoded_sentence[posit] = rand_syn;
            except KeyError:
                pass;
    return encoded_sentence;
    
augment_sentence(example_test_sequence);