Phishtank XML parsing (as HTML because the XML is broken)

phishtank_xml.py

import lxml.etree as etree


class CollectorTarget(object):
    def __init__(self):
        self.events = []
        self.urls = 0
    def start(self, tag, attrib):
        #self.events.append("start %s %r" % (tag, dict(attrib)))
        if tag == 'url':
            self.urls += 1
    def end(self, tag):
        return
        self.events.append("end %s" % tag)
    
    def data(self, data):
        return
        self.events.append("data %r" % data)

    def comment(self, text):
        return
        self.events.append("comment %s" % text)

    def close(self):
        return "closed!"

f = file('phishtank.xml')

target = CollectorTarget()
parser = etree.HTMLParser(recover=True, target=target)
tree   = etree.parse(f, parser)

print target.urls

The trick is in etree.HTMLParser(recover=True, target=target), don't use XMLParser since the xml file is invalid/broken and things won't work.