Scrapy Spider Middleware

synchronous.py

from scrapy import Request

class SynchronousSpiderMiddleware(object):


	def process_spider_output(self, response, result, spider):

		listing_request = None
		for request in result:
			step = request.meta.get('step', None)

			if step == 'parse':
				if not listing_request:
					listing_request = request

				listings = listing_request.meta.get('listings', [])
				listings.append(request)
				continue

			elif step == 'result':

				listings = request.meta.get('listings', [])
				request = listings.pop(0)
				yield request

		if step == 'parse':
			yield listing_request