gistfile1.py

import csv
from cStringIO import StringIO

from scrapy.http import Request
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector

from scrapy_tests.items import CrunchfeedItem

class Crunchfeed(BaseSpider):
    name = "crunchfeed"
    start_urls = ["file:///tmp/items.csv"]

    def parse(self, response):
        for row in csv.DictReader(StringIO(response.body)):
            cookiejar = "%s" % hash(row["url"])
            yield Request(row["url"], callback=self.parse_site, meta={"cookiejar": cookiejar})
            yield Request(row["blog"], callback=self.parse_site, meta={"cookiejar": cookiejar})

    def parse_site(self, response):
        hxs = HtmlXPathSelector(response)
        return CrunchfeedItem(
            title=''.join(hxs.select("//title/text()").extract())
        )