parse_arxiv_data.py

import glob

entries = []
for n,fname in enumerate(glob.glob('/Users/tom/Downloads/data/*/*.txt')):
    f = open(fname)
    s = f.readlines()
    x = [g.split('\t') for g in ' '.join(s).strip().split('\n ----------\n')][:-1]
    if n%1000 == 0:
        print n,fname
    for raw_entry in x:
        entry = {}
        entry['id'] = raw_entry[0].strip()
        entry['sections'] = raw_entry[1].strip().split(' ')
        entry['title'] = raw_entry[2].strip().replace('\n','').replace('   ',' ')
        entry['abstract'] = raw_entry[3].strip().replace('\n','')
        entry['date'] = fname.split('/')[-1][:-4]
        entries.append(entry)