Skip to content

Instantly share code, notes, and snippets.

@stealthbomber10
Created December 4, 2019 19:25
Show Gist options
  • Save stealthbomber10/9978e8c19faa3c58db139b3581fc9d6a to your computer and use it in GitHub Desktop.
Save stealthbomber10/9978e8c19faa3c58db139b3581fc9d6a to your computer and use it in GitHub Desktop.
scraping contact info from some real estate broker site
import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
driver = webdriver.Firefox(executable_path='C:\\Users\\steal\\workplace\\scrape\\geckodriver.exe')
for i in range(1, 24):
results_page = 'https://www.loopnet.com/commercial-real-estate-brokers/{}?searchbrokertype=1&propertyType=Multifamily&market=Tulsa&country=US'.format(i)
soup = BeautifulSoup(requests.get(results_page, headers={"User-Agent": "Mozilla/5.0"}).text, 'html.parser')
for broker_detail in soup.find_all('div', class_='broker-detail'):
broker_detail = broker_detail.find('h4').find('a')
broker_name = broker_detail.string
broker_link = 'https://www.loopnet.com/{}'.format(broker_detail.get('href'))
driver.get(broker_link)
driver.find_element_by_link_text('Send a Message').click()
phone = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.ID, 'broker-phone-number'))).text
print('{} {}'.format(broker_name, phone))
driver.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment