# -*- coding: utf-8 -*- | |
# See documentation in: | |
# https://doc.scrapy.org/en/latest/topics/commands.html#custom-project-commands | |
from scrapy.commands import ScrapyCommand | |
from scrapy.crawler import CrawlerProcess | |
from scrapy.utils.project import get_project_settings | |
class Command(ScrapyCommand): | |
requires_project = True | |
def short_desc(self): | |
return 'Run all spiders' | |
def run(self, args, opts): | |
process = CrawlerProcess(get_project_settings()) | |
for s in sorted(self.crawler_process.spider_loader.list()): | |
process.crawl(s) | |
process.start() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment