Skip to content

Instantly share code, notes, and snippets.

@TGOlson
Last active August 17, 2019 15:31
Show Gist options
  • Save TGOlson/c3f64c7b57a4b3db26d33447d627c3eb to your computer and use it in GitHub Desktop.
Save TGOlson/c3f64c7b57a4b3db26d33447d627c3eb to your computer and use it in GitHub Desktop.
yahoo draft value scraper
// setup
// install if you don't have the below packages already
//
// $ npm install node-fetch --save
// $ npm install jsdom --save
// imports
const fs = require('fs');
const fetch = require('node-fetch');
const jsdom = require("jsdom");
const { JSDOM } = jsdom;
// config
const pages = [
['QB', 0],
['WR', 0],
['WR', 1],
['RB', 0],
['RB', 1],
['TE', 0],
];
const headers = [
'Position',
'Name',
'Projected',
'Average'
];
const outputFile = './yahoo_values.csv';
// utils
const baseUrl = 'https://football.fantasysports.yahoo.com/f1';
const staticSuffix = 'draftanalysis?tab=AD&sort=DA_AP';
const makeUrl = (position, page) => (
`${baseUrl}/${staticSuffix}&pos=${position}&count=${page * 50}`
);
const toArray = (el) => Array.prototype.slice.call(el);
// parsing
const parseRowsFromDocument = (document) => {
const rows = toArray(document.querySelectorAll('#draftanalysistable tbody tr'));
return parsedDetails = rows.map(row => {
const name = row.querySelector('.name').text;
const valueElements = toArray(row.querySelectorAll('td')).slice(1, 3);
const [projected, averageRaw] = valueElements.map(el => el.querySelector('div').innerHTML);
const average = averageRaw === '$-' ? '$0' : averageRaw
return [name, projected, average];
});
}
const fetchCsvDataset = ([position, page]) => (
fetch(makeUrl(position, page))
.then(res => res.text())
.then(body => {
const dom = new JSDOM(body);
const { document } = dom.window;
const rows = parseRowsFromDocument(document);
return rows.map(x => position + ',' + (x === '$-' ? '$0' : x).join(',')).join('\n');
})
)
const main = () => Promise.all(pages.map(fetchCsvDataset))
.then(datasets => {
const csv = [headers, ...datasets].join('\n');
fs.writeFile(outputFile, csv, (err) => {
if (err) console.log('error!', err);
console.log('success')
})
});
main();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment