A Python script to scrape the Craigslist apartment listings into an Excel spreadsheet.

CraigslistScraper.py

import requests
import bs4
import openpyxl

#Functions to remove empty spaces and make a dictionary of listing names/URLs
def strip(txt):
    ret=""
    for l in txt.split("\n"):
        if l.strip()!='':
            ret += l + "\n"
    return ret

listings = dict()
def addListing(desc, url):
    if desc in listings:
        print('Duplicate')
    else:
        listings[desc] = url

def getSearchPageListings(url):
    res = requests.get(url)
    res.raise_for_status()
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    for a in soup.find_all("a", class_="result-title hdrlnk"):
        addListing(a.string, a.get('href'))

#STEP 1: GET RESULTS URLS
a_url='https://miami.craigslist.org/search/mdc/sub?max_price=900&bundleDuplicates=1&hasPic=1&min_price=400'
getSearchPageListings(a_url)


bodysum = dict()
bodyText = dict()

def MineListing(listUrl):
    res = requests.get(listUrl)
    res.raise_for_status()
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    desc = strip(soup.get_text())
    if (desc.find("google map") == -1):
        print('No map.')
    else:
        bodysum[listUrl] = desc[desc.find("google map")+12:desc.find("QR Code Link to This Post")]
        bodyText[listUrl] = desc[desc.find("QR")+25:desc.find("email to friend")]
    

#STEP 2: GET DESCRIPTIONS FROM THOSE PAGES
for key in listings:
    MineListing(listings[key])

#Step 3: PUSH RESULTS INTO EXCEL
wb = openpyxl.Workbook()
ws = wb.create_sheet("Listings", 0)

i=1
for key in bodyText:
    ws.cell(row=i, column=2).value = bodysum[key]
    ws.cell(row=i, column=3).value = bodyText[key]
    i += 1

wb.save("C:/Users/Desktop/Craigslist.xlsx")

print('Done.')