prepare ner data for multitask learning pipe

load_ner_data.py

def load_ner_data(ner_path, seq_len=24):
    data = pd.read_csv(ner_path, encoding= 'unicode_escape', sep=',')
    data = data.fillna(method='ffill')
    grouped_s = data.groupby('Sentence #', as_index=True)['Word'].apply(lambda g: ' '.join(g))
    grouped_t = data.groupby('Sentence #',  as_index=True)['Tag'].apply(lambda g: ' '.join(g))

    ner_tr = pd.DataFrame({}, columns=['sentence', 'tag'] )
    ner_tr['sentence'] = [st for st in grouped_s.values if len(st.split())<=seq_len]
    ner_tr['tag'] = [ tg.split() for tg in grouped_t if len(tg.split())<=seq_len]

    tag2idx = {t: i for i,t in enumerate(data.Tag.unique())}
    num_tags = len(tag2idx)

    y = [[tag2idx[w] for w in s] for s in ner_tr['tag']]
    y = pad_sequences(maxlen = seq_len, sequences=y, padding='post', value=tag2idx["O"])

    ptargs = [to_categorical(i, num_classes=num_tags) for i in y]        
    ptexts = np.array(ner_tr['sentence'])

    return ptexts, ptargs, num_tags, tag2idx, ner_tr