crawlall.py

# -*- coding: utf-8 -*-

# See documentation in:
# https://doc.scrapy.org/en/latest/topics/commands.html#custom-project-commands

from scrapy.commands import ScrapyCommand
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

class Command(ScrapyCommand):

    requires_project = True

    def short_desc(self):
        return 'Run all spiders'

    def run(self, args, opts):
        process = CrawlerProcess(get_project_settings())
        for s in sorted(self.crawler_process.spider_loader.list()):
            process.crawl(s)
        process.start()