asyncspider.py

import scrapy
import asyncio
import aioredis


class MySpider(scrapy.Spider):
  name = "asyncspider"
  
  async def start_requests(self):
    self.redis = await aioredis.create_connection("redis://localhost")
    for url in self.start_urls:
      yield scrapy.Request(url, dont_filter=True)
      
  async def parse(self, response):
    for link in response.css("a"):
      yield response.follow(link)
      
    title = response.css("h1::text").get()
    res = await scrapy.Request(f"http://google.com/?q={title}")
    
    for item in await self.get_items(res):
      yield item
     
    return {"FIN": "THEEND"}

  async def get_items(self, response):
    for key in response.css(".keys::text").extract()
      await asyncio.sleep(0.1)
      yield {
        "data": await self.redis.execute("get", key),
        "url": response.url,
      }