gistfile1.py

from scrapy.spider import Spider
from scrapy.http import Request
from scrapy.selector import Selector


class CampUSA(Spider):
	"""
	This spider iterates through the list of states on the homepage, and plunges into each one. It then blindly clicks search to bring up ALL results for that state.
	"""
	name = 'camp'
	allowed_domains = ['campusa.com']
	start_urls = [
		'http://www.campusa.com',
	]

	def parse(self, response):
		sel = Selector(response)
		states = [sel.xpath('/html/body/table/tbody/tr/td/table/tbody/tr[1]/td/table/tbody/tr[6]/td[2]/table/tbody/tr/td[2]/li['+str(num)+']/a/@href').extract() for num in range(1, 51)]
		print states
		yield (Request(state, self.parse) for state in states)