Skip to content

Instantly share code, notes, and snippets.

@anarchivist

anarchivist/off-campus-housing-feed.py

Created December 22, 2009 06:34
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save anarchivist/261553 to your computer and use it in GitHub Desktop.
Save anarchivist/261553 to your computer and use it in GitHub Desktop.
Download ZIP
Screenscrape listings from Yale University's off-campus housing site and create an RSS feed
Raw
off-campus-housing-feed.py
#!/usr/bin/env python
# Gets listings from Yale University's off-campus housing site and generates
# feeds, if you're into that sort of thing.
import sys
import time
import urllib2
from BeautifulSoup import BeautifulSoup
from feedformatter import Feed
BASE_URL = 'http://www-iisp1.its.yale.edu/offcampus/'
SOURCE_URL = BASE_URL + 'offcampus_housing_results.asp'
BASE_DATA = 'rb_Orderby=DateListed&cmd_Search=Search'
def parse_field(field):
if field.find('a') is not None:
url = field.find('a')['href']
return BASE_URL + url.split('&')[0]
else:
return field.string
def row_to_dict(row):
item = {}
item['title'] = row[0] + ', ' + row[1] + ' ($' + row[5] + ')'
item['link'] = row[9]
item['guid'] = row[9]
item['description'] = '''<ul>
<li><strong>%s Term, %s Room %s</strong></li>
<li><strong>Handicap Accessible:</strong> %s</li>
<li><strong>Pets Allowed:</strong> %s</li>
<li><strong>Furnished:</strong> %s</li>
</ul>''' % (row[2], row[4], row[3], row[6], row[7], row[8])
item['pubDate'] = time.localtime()
return item
def get_listings(min_rent=0, max_rent=5000):
postdata = 'lb_rentStart=%s&lb_rentEnd=%s&%s' % \
(min_rent, max_rent, BASE_DATA)
listings = BeautifulSoup(urllib2.urlopen(SOURCE_URL, postdata).read())
listing_block = listings.find('table', width='750', border='1')
rows = listing_block.findAll('tr')
# pop the first, since it's just headings
rows.pop(0)
# Return the following list for each row:
# [Address, City, Term, Type, Beds, Rent, Handicap, Pets, Furnished, URL]
return [[parse_field(f) for f in row.findAll('td')] for row in rows]
def listings_to_feedobj(listing_matrix):
feed = Feed()
feed.feed['title'] = 'Yale Off-Campus Housing Listings (Unofficial Feed)'
feed.feed['link'] = SOURCE_URL
feed.feed['author'] = 'Elihu Yale'
feed.feed['description'] = 'Listings from Yale\'s Off-Campus Housing Site'
feed.items = [row_to_dict(row) for row in listing_matrix]
return feed
if __name__ == '__main__':
feedobj = listings_to_feedobj(get_listings())
if len(sys.argv) > 1:
out = sys.argv[1]
else:
out = 'yale-listings.xml'
feedobj.format_rss2_file(out)
Raw
yale-listings.sample.xml
<!-- sample output -->
<rss version="2.0"><channel><title>Yale Off-Campus Housing Listings (Unofficial Feed)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_results.asp</link><description>Listings from Yale's Off-Campus Housing Site</description><item><title>708 Orange St., New Haven ($1200)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27141</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 3 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27141</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>783-A Mix Avenue, Hamden ($1192)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27122</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27122</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>Short Beach, Branford ($800)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27142</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 1 Room Share&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27142</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>140 Thompson Street #26F, East Haven ($1400)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27146</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Condo&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27146</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>304 Humphrey Street, New Haven ($1200)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27145</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Short Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27145</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>548 Orange Street Apt. 403, New Haven ($1750)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27144</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27144</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>214-216 Central Ave, New Haven ($1100)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27143</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27143</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>9 Beecher Rd, Woodbridge ($700)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27140</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 1 Room Room&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; Yes&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27140</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>133-B Howe St, New Haven ($595)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27139</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Short Term, E Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27139</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>73 Nicoll St, New Haven ($1170)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27138</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27138</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>75 Nash St, New Haven ($891)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27137</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27137</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>108 Burban Dr, Branford ($1250)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27136</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27136</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>123 York Street, New Haven ($1425)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27135</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 1 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; Yes&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27135</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>129 York Street, New Haven ($1045)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27134</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 1 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27134</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>111 Park St, New Haven ($925)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27133</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, E Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; Yes&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27133</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>267 Humphrey St, New Haven ($860)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27132</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 1 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27132</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>123 York St, New Haven ($1150)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27131</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 1 Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; Yes&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27131</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>George &amp; College Streets, New Haven ($845)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27130</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, E Room Apt&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; Possible&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27130</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>Audobon St, New Haven ($2300)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27129</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 2 Room Condo&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; No&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27129</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item><item><title>74 East Overshore Drive, Madison ($1100)</title><link>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27128</link><description>&lt;ul&gt;
&lt;li&gt;&lt;strong&gt;Long Term, 3 Room Share&lt;/strong&gt;&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Handicap Accessible:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Pets Allowed:&lt;/strong&gt; No&lt;/li&gt;
&lt;li&gt;&lt;strong&gt;Furnished:&lt;/strong&gt; Yes&lt;/li&gt;
&lt;/ul&gt;</description><guid>http://www-iisp1.its.yale.edu/offcampus/offcampus_housing_details.asp?HL_PK=27128</guid><pubDate>Tue, 22 Dec 2009 13:01:02 UTC</pubDate></item></channel></rss>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment