Skip to content

Instantly share code, notes, and snippets.

@pastleo
Created May 28, 2016 06:31
Show Gist options
  • Save pastleo/9351ba705693dd97640f7e268697cda8 to your computer and use it in GitHub Desktop.
Save pastleo/9351ba705693dd97640f7e268697cda8 to your computer and use it in GitHub Desktop.
freecodecamp crawler using promise, generator and coroutine
var request = require('request-promise');
var co = require('co');
var fs = require('fs');
var cheerio = require('cheerio');
var dateFormat = require('dateformat');
var usernames = ['jd615645', 'jayhung97724'];
var fcc_info = [];
var parser_time = dateFormat(new Date(), 'yyyy-mm-dd HH:MM:ss');
co(function*(){
var fcc_info = yield usernames.map(function(name) {
return request({
url: 'https://www.freecodecamp.com/' + name,
method: 'GET'
});
});
return fcc_info.map(function(data,i) {
$ = cheerio.load(data);
var source = parseInt($('h1.text-primary').text().match(/[0-9]{2}/));
var challenges = $('table tr');
var challenge = [];
for (var j = 0; j < challenges.length; j++) {
var name = $(challenges[j]).find('td:first-child').text();
var time = $(challenges[j]).find('td:nth-child(2)').text();
challenge.push({
name: name,
time: time
});
}
return {
username: usernames[i],
source: source,
challenges: challenge
};
});
}).then(
fcc_info => fs.writeFileSync("result.json", JSON.stringify(fcc_info)),
err => console.error(err)
);
{
"name": "try_promise",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"cheerio": "^0.20.0",
"co": "^4.6.0",
"dateformat": "^1.0.12",
"request-promise": "^3.0.0"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment