s5.py

WINDOW_SIZE = 5
NEGATIVE_SAMPLES = 10

def process_sent(sent):
    sampling = keras.preprocessing.sequence.make_sampling_table(VOCAB_SIZE)
    couples, labels = keras.preprocessing.sequence.skipgrams(
        sequence=sent, vocabulary_size=VOCAB_SIZE, window_size=WINDOW_SIZE,
        negative_samples=NEGATIVE_SAMPLES, shuffle=True, sampling_table=sampling
    )
      target, context = zip(*couples)
      target = list(target)
      context = list(context)
      return [target, context, labels]

def process_batch(data):
    with Pool(processes=NUM_WORKERS) as pool:
        results = pool.map(process_sent, data, chunksize=200)
    results = [x for x in results if x is not None]
    return np.asarray(results)

train_input = process_batch(train_text_ids)
test_input = process_batch(test_text_ids)