Skip to content

Instantly share code, notes, and snippets.

View gitdagray's full-sized avatar
👋
☕ 💻 Coffee and Code

Dave Gray gitdagray

👋
☕ 💻 Coffee and Code
View GitHub Profile
@gitdagray
gitdagray / kanview_selenium4.py
Last active February 18, 2018 18:46
Converting pandas dataframes to JSON
#loop has completed
#end the Selenium browser session
driver.quit()
#combine all pandas dataframes in the list into one big dataframe
result = pd.concat([pd.DataFrame(datalist[i]) for i in range(len(datalist))],ignore_index=True)
#convert the pandas dataframe to JSON
json_records = result.to_json(orient='records')
@gitdagray
gitdagray / kanview_selenium3.py
Last active February 18, 2018 18:08
Scraping Kanview with Selenium, Beautiful Soup, and pandas
#Beautiful Soup grabs all Job Title links
for link in soup_level1.find_all('a', id=re.compile("^MainContent_uxLevel2_JobTitles_uxJobTitleBtn_")):
#Selenium visits each Job Title page
python_button = driver.find_element_by_id('MainContent_uxLevel2_JobTitles_uxJobTitleBtn_' + str(x))
python_button.click() #click link
#Selenium hands of the source of the specific job page to Beautiful Soup
soup_level2=BeautifulSoup(driver.page_source, 'lxml')
@gitdagray
gitdagray / kanview_selenium2.py
Created February 18, 2018 17:43
Scraping Kanview with Selenium and Beautiful Soup
#Selenium hands the page source to Beautiful Soup
soup_level1=BeautifulSoup(driver.page_source, 'lxml')
datalist = [] #empty list
x = 0 #counter
for link in soup_level1.find_all('a', id=re.compile("^MainContent_uxLevel2_JobTitles_uxJobTitleBtn_")):
##code to execute in for loop goes here
@gitdagray
gitdagray / kanview_selenium1.py
Created February 18, 2018 17:37
Launching Selenium on Kanview
#launch url
url = "http://kanview.ks.gov/PayRates/PayRates_Agency.aspx"
# create a new Firefox session
driver = webdriver.Firefox()
driver.implicitly_wait(30)
driver.get(url)
python_button = driver.find_element_by_id('MainContent_uxLevel1_Agencies_uxAgencyBtn_33') #FHSU
python_button.click() #click fhsu link
@gitdagray
gitdagray / Python_Selenium_Soup_Import.py
Created February 18, 2018 17:29
Python importing Selenium, Beautiful Soup, RegEx, and pandas.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
@gitdagray
gitdagray / Python_SeleniumImport.py
Created February 18, 2018 17:24
Importing Selenium into Python
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
@gitdagray
gitdagray / office_hours.py
Created November 2, 2017 17:26
Python Office Hours: A simple example with my office hours... that compares user input to dictionary keys and returns the corresponding value. By making the comparison with get(value_to_compare, default_value), the default value of "by appointment" is returned if the user input does not match one of the dictionary keys.
while True:
day_in_question = input("\nIn order to see my office hours for a given day, " + \
"\n...please enter the specific day or day abbreviation: ").capitalize()
print("\nThank you for entering: {0}.".format(day_in_question))
if (day_in_question[0] == "M" or
day_in_question[0] == "T" or
day_in_question[0] == "W" or