stackoverflow-salary-scrape.py

# import libraries
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
import time
import pandas as pd

# add chromedriver options
options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')

# define main function
def search_combination(loc, ed, exp):
    driver = webdriver.Chrome(executable_path="/Users/erikgregorywebb/Downloads/chromedriver 2", chrome_options=options)
    link = 'https://stackoverflow.com/jobs/salary'
    driver.get(link)
    
    time.sleep(1)
    
    # job title
    select = Select(driver.find_element_by_css_selector('#dr'))
    select.select_by_visible_text('Data Scientist')
    
    # location
    element = driver.find_element_by_css_selector('#l')
    element.clear()
    element.send_keys(loc)

    # education
    select = Select(driver.find_element_by_css_selector('#ed'))
    select.select_by_index(ed)

    # expereince
    element = driver.find_element_by_css_selector('#ex')
    element.send_keys(exp)

    time.sleep(3)

    # click submit
    element = driver.find_element_by_css_selector('.js-trigger')
    element.click()

    time.sleep(4)

    # result
    sal_25 = driver.find_element_by_css_selector('.salary-result-25th span:nth-child(2)').text
    sal_50 = driver.find_element_by_css_selector('.salary-result-50th span:nth-child(2)').text
    sal_75 = driver.find_element_by_css_selector('.salary-result-75th span:nth-child(2)').text

    sals = [loc, ed, exp, sal_25, sal_50, sal_75]
    driver.close()
    return(sals)

# function inputs
loc_options = ['New York City, NY, United States', 'Salt Lake City, UT, United States', 'Houston, TX, United States', 'San Francisco, CA, United States', 'San Jose, CA, United States', 'Los Angeles, CA, United States', 'Chicago, IL, United States', 'Phoenix, AZ, United States', 'Boston, MA, United States']
ed_options = [1, 2, 3, 4]
exp_options = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10']

# loop through inputs
sals = []
for i in range(0, len(loc_options)):
    for j in range(0, len(ed_options)):
        for k in range(0, len(exp_options)):
            time.sleep(3)
            sal = search_combination(loc_options[i], ed_options[j], exp_options[k])
            print(sal)
            sals.append(sal)
            
# export
so_salaries = pd.DataFrame(sals)
so_salaries.to_csv('so-salaries.csv', index = False)
#