spider.py

def parse_detail(self, response):
		#parsing agent for the actual home detail we are interested in
		addr_bbs = response.xpath('//h3/span[@class="addr_bbs"]/text()').extract()
		addr1 = response.xpath('//h1[@class="notranslate"]/text()').extract()
		addr2 = response.xpath('//h1[@class="notranslate"]/span/text()').extract()
		
		item = PropertyItem()
		
		item['address1'] = ''.join(addr1)
		item['address2'] = ''.join(addr2)
		#item['price'] = 


		#getting around props who dont store bed/bath/sqft ; this should be handled
		#more properly
		if addr_bbs:
			item['bedrooms'] = addr_bbs[0]
			item['bathrooms'] = addr_bbs[1]
			item['sqft'] = addr_bbs[2]
		else:
			item['bedrooms'] = ""
			item['bathrooms'] = ""
			item['sqft'] = ""

		#prop status flag
		item['status'] = ''.join(response.xpath('//div[contains(@class, "status-icon-row")]/text()').extract()).strip()
		
		if item['status'] == 'Sold:':
			item['sale_price'] = ''.join(response.xpath('//div[contains(@class, "status-icon-row")]/span[not(contains(@id, "listing-icon"))]/text()').extract()).strip()
			item['sale_date'] = response.xpath('//*[text()[contains(.,"Sold on")]]/span/text()').extract()[0].strip()
		if item['status'] in 'For Sale':
			item['listing_price'] = response.xpath('//div[contains(@class, "main-row")]/span/text()').extract_first().strip()

		yield item