Skip to content

Instantly share code, notes, and snippets.

@octoparse octoparse/
Created Nov 11, 2019

What would you like to do?
Scraping fantasy football projections
from bs4 import BeautifulSoup
import re
import requests
def get_html_data(url):
response = requests.get(url)
return BeautifulSoup(response.content, "html5lib")
def scrape(return_list=None):
url = '{0}&position=1&sort=projectedPts&statCategory=projectedStats&statSeason=2019&statType=seasonProjectedStats&statWeek=10'
regex = re.compile('player-.+')
regex2 = re.compile('playerCard playerName playerNameFull playerNameId-.+')
num = 1
while num < 139:
_url = url.format(num)
soup = get_html_data(_url)
trs = soup.find_all('tr', {'class': regex})
for tr in trs:
name = tr.find('a', {'class': regex2}).text
a = tr.find('td', {'class': 'stat stat_1 numeric'}).text
b = tr.find('td', {'class': 'stat stat_5 numeric'}).text
c = tr.find('td', {'class': 'stat stat_6 numeric'}).text
d = tr.find('td', {'class': 'stat stat_7 numeric'}).text
e = tr.find('td', {'class': 'stat stat_14 numeric'}).text
f = tr.find('td', {'class': 'stat stat_15 numeric'}).text
g = tr.find('td', {'class': 'stat stat_20 numeric'}).text
h = tr.find('td', {'class': 'stat stat_21 numeric'}).text
i = tr.find('td', {'class': 'stat stat_22 numeric'}).text
j = tr.find('td', {'class': 'stat stat_28 numeric'}).text
k = tr.find('td', {'class': 'stat stat_29 numeric'}).text
l = tr.find('td', {'class': 'stat stat_32 numeric'}).text
m = tr.find('td', {'class': 'stat stat_30 numeric'}).text
n = tr.find('td', {'class': 'stat projected numeric sorted last'}).text
if return_list:
return_list.append(name, a, b, c, d, e, f, g, h, i, j, k, l, m, n)
print '{}, {},{},{},{},{},{},{},{},{},{},{},{},{},{}'.format(name, a, b, c, d, e, f, g, h, i, j, k, l, m, n)
num += 25
def main():
return scrape()
if __name__ == '__main__':
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.