split2sentences.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import codecs
import glob
import os
import re
import sys

import nltk.data

pickle = nltk.data.load('tokenizers/punkt/english.pickle')

dir_name = 'sentences'

if not os.path.exists(dir_name):
    os.mkdir(dir_name)

for txt_filename in glob.glob("txt/*.txt"):
    sent_filename = os.path.join(dir_name, os.path.basename(txt_filename))
    
    print txt_filename.encode('ascii', 'ignore')
    
    with codecs.open(txt_filename, 'r', 'utf-8') as f_txt, codecs.open(sent_filename, 'w', 'utf-8') as f_sent:
        data = f_txt.read()
        txt = data.replace(". . .", "...")

        lines = txt.splitlines()
        for idx, line in enumerate(lines):
            line = line.strip()

            for s in pickle.tokenize(line):
                f_sent.write(s)
                f_sent.write("\n")