timeout schedule middleware in scrapy

timeout_schedule_middleware.py

class TimeoutScheduleMiddleware(object):
    TIMEOUT_SEC = 3600

    def __init__(self, crawler):
        self.crawler = crawler
        self.scheduler_per_spider = {}

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls(crawler)
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def spider_opened(self, spider):
        self.scheduler_per_spider[spider.name] = time.time()

    def process_request(self, request, spider):
        elapsed = time.time() - self.scheduler_per_spider.get(spider.name, self.TIMEOUT_SEC)
        if elapsed > self.TIMEOUT_SEC:
            self.scheduler_per_spider[spider.name] = time.time()
            self.crawler.engine.close_spider(spider, 'closespider_time_limit_exceeded')