run_spider.py

__author__ = 'yetone'

from twisted.internet import reactor

from scrapy import log, signals
from scrapy.crawler import Crawler
from scrapy.xlib.pydispatch import dispatcher
from scrapy.utils.project import get_project_settings

from araneae.spiders.aliexpress.item_list import ItemListSpider


def stop_reactor():
    reactor.stop()

dispatcher.connect(stop_reactor, signal=signals.spider_closed)
spider = ItemListSpider(url='http://activities.aliexpress.com/bestselling.php')
crawler = Crawler(get_project_settings())
crawler.configure()
crawler.crawl(spider)
crawler.start()
log.start()
log.msg('Running reactor...')
reactor.run()  # the script will block here until the spider is closed
log.msg('Reactor stopped.')