This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#loop has completed | |
#end the Selenium browser session | |
driver.quit() | |
#combine all pandas dataframes in the list into one big dataframe | |
result = pd.concat([pd.DataFrame(datalist[i]) for i in range(len(datalist))],ignore_index=True) | |
#convert the pandas dataframe to JSON | |
json_records = result.to_json(orient='records') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#Beautiful Soup grabs all Job Title links | |
for link in soup_level1.find_all('a', id=re.compile("^MainContent_uxLevel2_JobTitles_uxJobTitleBtn_")): | |
#Selenium visits each Job Title page | |
python_button = driver.find_element_by_id('MainContent_uxLevel2_JobTitles_uxJobTitleBtn_' + str(x)) | |
python_button.click() #click link | |
#Selenium hands of the source of the specific job page to Beautiful Soup | |
soup_level2=BeautifulSoup(driver.page_source, 'lxml') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#Selenium hands the page source to Beautiful Soup | |
soup_level1=BeautifulSoup(driver.page_source, 'lxml') | |
datalist = [] #empty list | |
x = 0 #counter | |
for link in soup_level1.find_all('a', id=re.compile("^MainContent_uxLevel2_JobTitles_uxJobTitleBtn_")): | |
##code to execute in for loop goes here |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#launch url | |
url = "http://kanview.ks.gov/PayRates/PayRates_Agency.aspx" | |
# create a new Firefox session | |
driver = webdriver.Firefox() | |
driver.implicitly_wait(30) | |
driver.get(url) | |
python_button = driver.find_element_by_id('MainContent_uxLevel1_Agencies_uxAgencyBtn_33') #FHSU | |
python_button.click() #click fhsu link |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from selenium import webdriver | |
from selenium.webdriver.common.keys import Keys | |
from bs4 import BeautifulSoup | |
import re | |
import pandas as pd | |
import os |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from selenium import webdriver | |
from selenium.webdriver.common.keys import Keys |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
while True: | |
day_in_question = input("\nIn order to see my office hours for a given day, " + \ | |
"\n...please enter the specific day or day abbreviation: ").capitalize() | |
print("\nThank you for entering: {0}.".format(day_in_question)) | |
if (day_in_question[0] == "M" or | |
day_in_question[0] == "T" or | |
day_in_question[0] == "W" or |
NewerOlder