Skip to content

Instantly share code, notes, and snippets.

@ceekz
Created Dec 13, 2016
Embed
What would you like to do?
# -*- coding: utf-8 -*-
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/commands.html#custom-project-commands
from scrapy.commands import ScrapyCommand
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
class Command(ScrapyCommand):
requires_project = True
def short_desc(self):
return 'Run all spiders'
def run(self, args, opts):
process = CrawlerProcess(get_project_settings())
for s in sorted(self.crawler_process.spider_loader.list()):
process.crawl(s)
process.start()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment