Skip to content

Instantly share code, notes, and snippets.

@seanbehan
Last active February 24, 2018 13:44
Show Gist options
  • Save seanbehan/6a71808079f297396d50 to your computer and use it in GitHub Desktop.
Save seanbehan/6a71808079f297396d50 to your computer and use it in GitHub Desktop.
using generator comprehensions in python to extract words from epub files
from glob import glob
from os.path import basename
from zipfile import ZipFile, is_zipfile
from re import findall
from nltk.corpus import words
ROOT_PATH = '/Volumes/USB20FD/downloads/*.epub'
WORDS = set(words.words())
groups = (group for group in zip(*(iter(glob(ROOT_PATH)),)*100))
zips = ((ZipFile(file) for file in group if is_zipfile(file)) for group in groups)
files = (((z.open(f.filename).read() for f in z.infolist() if '.html' in f.filename) for z in zip) for zip in zips)
texts = ((((WORDS & set(findall(r'[\w]+', t.lower()))) for t in txt) for txt in text) for text in files)
corpus = set()
for text in texts:
for txt in text:
for t in txt:
corpus = (corpus | t)
print (float(len(corpus)) / float(len(WORDS)))*100
print "-"*75
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment