example

gistfile1.py

class DmozSpider(BaseSpider):
    name = "twitter.com"
    name = "dmoz"
    allowed_domains = ["codinginmysleep.com"]
    start_urls = [
        "http://codinginmysleep.com"
        # "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
    ]

    def parse(self, response):
        #logging.warning(response.body)
        hxs = HtmlXPathSelector(response)
        links = hxs.select("//a/@href")
        items = []

        for link in links:
            text = link.extract()

            getLinks(text)

            item = BlogscrapeItem()
            item['link'] = text
            items.append(item)

        return items