s3.py

def clean_sentence(dic):
    sent = dic["text"].decode("utf-8").lower().translate(EXCLUDE_TRANS)
    words = [LEMMA.lemmatize(word, POS_DIC[pos_tag([word])[0][1]]) for word in sent.split()
             if word.isalpha()]
    return words

NUM_WORKERS = 20
def clean_batch(batch_rows):
    with Pool(processes=NUM_WORKERS) as pool:
        batch = pool.map(clean_sentence, batch_rows, chunksize=200)
    return batch

train_text = clean_batch(np_train_data)
test_text = clean_batch(np_test_data)