Skip to content

Instantly share code, notes, and snippets.

@awesomephant
Created March 23, 2017 20:00
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save awesomephant/83ad6c9955d9fb1c7cba86a29ef558f0 to your computer and use it in GitHub Desktop.
Save awesomephant/83ad6c9955d9fb1c7cba86a29ef558f0 to your computer and use it in GitHub Desktop.
Casper.js script to load chess openings
var fs = require('fs')
var casper = require('casper').create();
casper.options.waitTimeout = 10000;
var games;
var count = 0;
var nextLink = 'http://www.chessgames.com/perl/chess.pl?tid=53788';
var parsePage = function () {
var rows = document.querySelectorAll('table[cellpadding="3"]:nth-of-type(2) tr');
var games = [];
for (var i = 1; i < rows.length; i++) {
var id = rows[i].querySelector('td:nth-child(7) font font a').innerHTML
var opening = rows[i].querySelector('td:nth-child(7) font font').innerHTML.replace(/<(?:.|\n)*?>/gm, '');
games.push(opening)
}
return games;
}
var getYear = function () {
var year = document.querySelectorAll('font[face^="georgia"] > font')[0].innerHTML.replace(/\D/g, '');
return year;
}
var getNextLink = function () {
var base = 'http://www.chessgames.com';
var links = document.querySelectorAll('[bgcolor="#FFE9BF"] a[href^="/perl/chess.pl"]');
if (links.length === 2 || links.length === 3 || links.length === 4) {
return base + links[1].getAttribute('href')
} else {
return base + links[0].getAttribute('href')
}
}
casper.start();
casper.repeat(46, function () {
console.log('Loading ' + nextLink + ' (#' + count + ')')
casper.thenOpen(nextLink, function () {
this.waitForSelector('table');
})
casper.then(function () {
games = this.evaluate(parsePage)
nextLink = this.evaluate(getNextLink)
year = this.evaluate(getYear)
fs.write('./data/wcc-' + year + '.json', JSON.stringify(games, null, ' '), 'w')
count++;
});
})
casper.on("page.error", function (msg, trace) {
this.echo("Error: " + msg, "ERROR");
});
casper.run(function () {
'\n All done.'
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment