Skip to content

Instantly share code, notes, and snippets.

@stav
Created August 14, 2015 19:56
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save stav/d680703bc4a68b1b7016 to your computer and use it in GitHub Desktop.
Save stav/d680703bc4a68b1b7016 to your computer and use it in GitHub Desktop.
Scrapy route spider pseudo code
class MySpider(RouteSpider):
name = "example.com"
def start_routes(self, response):
for city in response.css('#location .dropdown-bg li'):
yield Route(
url=city.xpath('a/@href'),
callback=self.parse_city,
)
def parse_city(self, response):
router = response.router
loader = router.loader
loader.add_value('url', response.url)
contact_url = response.xpath('li/a[contains(@href,"contact")]/@href')
saleses_url = response.xpath('li/a[contains(@href,"sales")]/@href')
# Add new routes
router.add_hop(url=contact_url, callback=self.parse_contact)
router.add_hop(url=saleses_url, callback=self.parse_sales)
def parse_contact(self, response):
loader = response.router.loader
loader.add_xpath('contanct_email', 'p[1]/a[contains(@href,"mailto")]')
def parse_sales(self, response):
loader = response.router.loader
loader.add_xpath('sales_email', 'p[1]/a[contains(@href,"mailto")]')
@stav
Copy link
Author

stav commented Aug 14, 2015

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment