load CONLL2003 dataset using Pandas

load_conll2003.py

import pandas as pd

def read_conll(filename):
    df = pd.read_csv(filename,
                    sep = ' ', header = None, keep_default_na = False,
                    names = ['TOKEN', 'POS', 'CHUNK', 'NE'],
                    quoting = 3, skip_blank_lines = False)
    df['SENTENCE'] = (df.TOKEN == '').cumsum()
    return df[df.TOKEN != '']

train_df = read_conll('conll2003/train.txt')
valid_df = read_conll('conll2003/valid.txt')
test_df = read_conll('conll2003/test.txt')