demo.py

settings = {}
bot = scrapy.CrawlerBot(name="mybot/1.0", settings=settings)

def follow_links(response):
     for link in response.iter_links():
         bot.crawl(link.url, callback=follow_links, referer=response)
         
     bot.emit({
         "url": response.url,
         "status": response.status,
     })
         
bot.crawl("http://google.com", callback=parse_google)

bot.start()