Instantly share code, notes, and snippets.

Embed
What would you like to do?
# import libraries
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
import time
import pandas as pd
# add chromedriver options
options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
# define main function
def search_combination(loc, ed, exp):
driver = webdriver.Chrome(executable_path="/Users/erikgregorywebb/Downloads/chromedriver 2", chrome_options=options)
link = 'https://stackoverflow.com/jobs/salary'
driver.get(link)
time.sleep(1)
# job title
select = Select(driver.find_element_by_css_selector('#dr'))
select.select_by_visible_text('Data Scientist')
# location
element = driver.find_element_by_css_selector('#l')
element.clear()
element.send_keys(loc)
# education
select = Select(driver.find_element_by_css_selector('#ed'))
select.select_by_index(ed)
# expereince
element = driver.find_element_by_css_selector('#ex')
element.send_keys(exp)
time.sleep(3)
# click submit
element = driver.find_element_by_css_selector('.js-trigger')
element.click()
time.sleep(4)
# result
sal_25 = driver.find_element_by_css_selector('.salary-result-25th span:nth-child(2)').text
sal_50 = driver.find_element_by_css_selector('.salary-result-50th span:nth-child(2)').text
sal_75 = driver.find_element_by_css_selector('.salary-result-75th span:nth-child(2)').text
sals = [loc, ed, exp, sal_25, sal_50, sal_75]
driver.close()
return(sals)
# function inputs
loc_options = ['New York City, NY, United States', 'Salt Lake City, UT, United States', 'Houston, TX, United States', 'San Francisco, CA, United States', 'San Jose, CA, United States', 'Los Angeles, CA, United States', 'Chicago, IL, United States', 'Phoenix, AZ, United States', 'Boston, MA, United States']
ed_options = [1, 2, 3, 4]
exp_options = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10']
# loop through inputs
sals = []
for i in range(0, len(loc_options)):
for j in range(0, len(ed_options)):
for k in range(0, len(exp_options)):
time.sleep(3)
sal = search_combination(loc_options[i], ed_options[j], exp_options[k])
print(sal)
sals.append(sal)
# export
so_salaries = pd.DataFrame(sals)
so_salaries.to_csv('so-salaries.csv', index = False)
#
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment