scrapy.py

from twisted.internet import reactor
from scrapy.crawler import Crawler
from scrapy.settings import CrawlerSettings
from scrapy import log as scrapy_log, signals


def _start_crawling(spider, log_level='INFO'):
    crawler = Crawler(CrawlerSettings(settings))
    crawler.configure()
    crawler.crawl(spider)
    crawler.start()
    crawler.signals.connect(reactor.stop, signal=signals.spider_closed)
    scrapy_log.start(loglevel=log_level)
    reactor.run()

def crawl_projects(urls, source, log_level='INFO'):
    spider = ProjectSpider(project_urls=urls)
    _start_crawling(spider, log_level)