Convert relevant html docs to text!

gistfile1.py

# open the files in the directory, parse them and get the text.


#eg.text_content()

import glob
from lxml import etree
import lxml.html
import csv   
import urllib2
import urllib
import re

flist = glob.glob('/Users/astorer/Work/sgrossman/output/*.html')
parser = etree.HTMLParser()

for fname in flist:   
    f = open(fname,'r')
    tree   = lxml.html.document_fromstring(f.read())
    f.close()
    ftname = fname.replace('.html','.txt')
    ft = open(ftname,'w')
    textelem = tree.xpath('//*[@id="myfile"]')[0]
    for el in textelem:      
        ft.write(el.text_content().encode('utf-8')+'\n')

There are details on the lxml.html module here: http://lxml.de/lxmlhtml.html