Skip to content

Instantly share code, notes, and snippets.

@alexmcpherson
Created September 17, 2012 00:52
Show Gist options
  • Star 9 You must be signed in to star a gist
  • Fork 5 You must be signed in to fork a gist
  • Save alexmcpherson/3735020 to your computer and use it in GitHub Desktop.
Save alexmcpherson/3735020 to your computer and use it in GitHub Desktop.
Casper.js vimrc scraper
var casper = require("casper").create();
var url = "https://github.com/search?langOverride=VimL&language=&q=vimrc&repo=&type=Repositories&start_value=" + casper.cli.get(0);
var repoLinks = [];
var fileLinks = [];
function getRepoLinks() {
var links = [];
$("div.results .result h2 a").each(function(i,el){
links.push(el.href);
});
return links
}
function findFile() {
var url = false;
$('td.content a').each(function(i,el){
if (el.innerHTML === 'vimrc' || el.innerHTML === 'gvimrc') {
url = el.href.replace('blob', 'raw');
}
});
return url;
}
casper.start(url, function() {
this.echo(url);
this.echo("Loaded search results page");
repoLinks = this.evaluate(getRepoLinks);
});
casper.then(function() {
this.each(repoLinks, function(self, link){
self.thenOpen(link, function() {
var fileLink = this.evaluate(findFile);
if (fileLink) fileLinks.push(fileLink)
})
});
});
casper.then(function(){
this.each(fileLinks, function(self, fileLink) {
self.thenOpen(fileLink, function() {
var segments = fileLink.split('/');
self.download(self.getCurrentUrl(), 'vimrcs/' + segments[segments.length - 5]);
});
});
});
casper.run();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment