Skip to content

Instantly share code, notes, and snippets.

@ThibaudLamothe
Last active February 1, 2021 11:14
Show Gist options
  • Save ThibaudLamothe/3b09856ab8aa20d9a6f935037e052aef to your computer and use it in GitHub Desktop.
Save ThibaudLamothe/3b09856ab8aa20d9a6f935037e052aef to your computer and use it in GitHub Desktop.
class airbnbSpider(scrapy.Spider):
name = "airbnbSpider"
def __init__(self):
# Urls of the cities to scrap
self.start_urls = [url_city_1, url_city_2]
# Trackers
self.page = 0
self.object = 0
def start_requests(self):
for url in self.start_urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
self.page += 1
logger.info(f'Search page N° {self.page}')
# Finding the links and following them (parse_hotel)
# Finding the next page and following it (parse)
def parse_hotel(self, response)
self.object += 1
logger.debug(f'Hotel N° {self.object}')
# Finding hotel information with selectors
# Yielding hotel information as dictionnary (or scrapy.Items)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment