Skip to content

Instantly share code, notes, and snippets.

@khunreus
Last active March 24, 2019 19:24
Show Gist options
  • Save khunreus/d7f7da31d68249d24343fa011a0cbc72 to your computer and use it in GitHub Desktop.
Save khunreus/d7f7da31d68249d24343fa011a0cbc72 to your computer and use it in GitHub Desktop.
"""
python3.6
Scrapy + Selenium
"""
for profile_url in profile_urls_distinct:
self.logger.info('Home #' + str(q))
self.driver.get(profile_url)
q = q+1
sleep(10)
link_to_home = profile_url
profile_scrapy_selector = Selector(text = self.driver.page_source)
property_name = profile_scrapy_selector.xpath('//*[@itemprop="name"]//h1/span/text()').extract()
price_night = profile_scrapy_selector.xpath('//*[@class ="_doc79r"]/text()').extract_first()
rating_overall = profile_scrapy_selector.xpath('//*[@itemprop="ratingValue"]/@content').extract_first()
rating_split = {}
rating_categories = profile_scrapy_selector.xpath('//*[@class="_iq8x9is"]/span/text()').extract()
rating_stars = profile_scrapy_selector.xpath('//*[@class="_iq8x9is"]/div/span//@aria-label').extract()
i = 0
for i in range(len(rating_categories)):
rating_split[rating_categories[i]] = rating_stars[i]
i = i+1
home_neighborhood_short = profile_scrapy_selector.xpath('//*[@class="_6z3til"]//*[@class="_czm8crp"]/span/text()').extract()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment