Skip to content

Instantly share code, notes, and snippets.

@erikgregorywebb
Last active February 21, 2022 04:59
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save erikgregorywebb/eaa441b988c2a6aaaa072fde38e069c9 to your computer and use it in GitHub Desktop.
Save erikgregorywebb/eaa441b988c2a6aaaa072fde38e069c9 to your computer and use it in GitHub Desktop.
import requests
from bs4 import BeautifulSoup
import pandas as pd
from datetime import datetime
# get page content
url = 'http://www.freddiemac.com/'
page = requests.get(url)
soup = BeautifulSoup(page.content, "html.parser")
# get current date and datetime
current_datetime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
current_datetime_label = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
# extract values
items = []
for grid in soup.find_all(class_="rate grid-x"):
name = grid.find(class_="name").text
rate = grid.find(class_="rate-percent").text
fees = grid.find(class_="fees").text
item = [name, rate, fees, current_datetime]
items.append(item)
# save as dataframe
df = pd.DataFrame(items)
df.columns = ['name', 'rate', 'fees', 'datetime']
# export to csv
file_path = 'data/fm-rates-' + current_datetime_label + '.csv'
df.to_csv(file_path, index = False)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment