Skip to content

Instantly share code, notes, and snippets.

View gitdagray's full-sized avatar
👋
☕ 💻 Coffee and Code

Dave Gray gitdagray

👋
☕ 💻 Coffee and Code
View GitHub Profile
@gitdagray
gitdagray / office_hours.py
Created November 2, 2017 17:26
Python Office Hours: A simple example with my office hours... that compares user input to dictionary keys and returns the corresponding value. By making the comparison with get(value_to_compare, default_value), the default value of "by appointment" is returned if the user input does not match one of the dictionary keys.
while True:
day_in_question = input("\nIn order to see my office hours for a given day, " + \
"\n...please enter the specific day or day abbreviation: ").capitalize()
print("\nThank you for entering: {0}.".format(day_in_question))
if (day_in_question[0] == "M" or
day_in_question[0] == "T" or
day_in_question[0] == "W" or
@gitdagray
gitdagray / Python_SeleniumImport.py
Created February 18, 2018 17:24
Importing Selenium into Python
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
@gitdagray
gitdagray / Python_Selenium_Soup_Import.py
Created February 18, 2018 17:29
Python importing Selenium, Beautiful Soup, RegEx, and pandas.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
@gitdagray
gitdagray / kanview_selenium1.py
Created February 18, 2018 17:37
Launching Selenium on Kanview
#launch url
url = "http://kanview.ks.gov/PayRates/PayRates_Agency.aspx"
# create a new Firefox session
driver = webdriver.Firefox()
driver.implicitly_wait(30)
driver.get(url)
python_button = driver.find_element_by_id('MainContent_uxLevel1_Agencies_uxAgencyBtn_33') #FHSU
python_button.click() #click fhsu link
@gitdagray
gitdagray / kanview_selenium2.py
Created February 18, 2018 17:43
Scraping Kanview with Selenium and Beautiful Soup
#Selenium hands the page source to Beautiful Soup
soup_level1=BeautifulSoup(driver.page_source, 'lxml')
datalist = [] #empty list
x = 0 #counter
for link in soup_level1.find_all('a', id=re.compile("^MainContent_uxLevel2_JobTitles_uxJobTitleBtn_")):
##code to execute in for loop goes here
@gitdagray
gitdagray / kanview_selenium3.py
Last active February 18, 2018 18:08
Scraping Kanview with Selenium, Beautiful Soup, and pandas
#Beautiful Soup grabs all Job Title links
for link in soup_level1.find_all('a', id=re.compile("^MainContent_uxLevel2_JobTitles_uxJobTitleBtn_")):
#Selenium visits each Job Title page
python_button = driver.find_element_by_id('MainContent_uxLevel2_JobTitles_uxJobTitleBtn_' + str(x))
python_button.click() #click link
#Selenium hands of the source of the specific job page to Beautiful Soup
soup_level2=BeautifulSoup(driver.page_source, 'lxml')
@gitdagray
gitdagray / kanview_selenium4.py
Last active February 18, 2018 18:46
Converting pandas dataframes to JSON
#loop has completed
#end the Selenium browser session
driver.quit()
#combine all pandas dataframes in the list into one big dataframe
result = pd.concat([pd.DataFrame(datalist[i]) for i in range(len(datalist))],ignore_index=True)
#convert the pandas dataframe to JSON
json_records = result.to_json(orient='records')
@gitdagray
gitdagray / kanview_selenium5.py
Created February 18, 2018 18:25
Creating the JSON data file.
#get current working directory
path = os.getcwd()
#open, write, and close the file
f = open(path + "\\fhsu_payroll_data.json","w") #FHSU
f.write(json_records)
f.close()
@gitdagray
gitdagray / kanview_complete.py
Last active June 10, 2020 18:22
Full Python code for web scraping Kanview utilizing Selenium, Beautiful Soup, and pandas
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
from tabulate import tabulate
import os
#launch url
url = "http://kanview.ks.gov/PayRates/PayRates_Agency.aspx"
@gitdagray
gitdagray / online_offline_detection.js
Last active March 14, 2024 20:55
Online / Offline Status Detection w/ Async & Await
/* ********** Online / Offline Detection ********** */
// Request a small image at an interval to determine status
// ** Get a 1x1 pixel image here: http://www.1x1px.me/
// ** Use this code with an HTML element with id="status"
const checkOnlineStatus = async () => {
try {
const online = await fetch("/1pixel.png");
return online.status >= 200 && online.status < 300; // either true or false