Last active
June 13, 2017 10:45
-
-
Save javilobo8/1ce6ac34ec828020c903f26739746902 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// npm i -S request request-promise cheerio lodash | |
const request = require('request-promise'); | |
const cheerio = require('cheerio'); | |
const fs = require('fs'); | |
const _ = require('lodash'); | |
const projectName = 'enaqx/awesome-react'; | |
// const projectName = 'vuejs/awesome-vue'; | |
const url = `https://raw.githubusercontent.com/${projectName}/master/README.md`; | |
const githubRegExp = /(http|https):\/\/github\.com\/[A-Za-z0-9_.\-]+\/[A-Za-z0-9_.\-]+/g; | |
function getMatched(data, regexp) { | |
const matches = []; | |
let match; | |
do { | |
match = regexp.exec(data) | |
if (match) { | |
matches.push(match[0].toLowerCase()); | |
} | |
} while (match); | |
console.log(matches.length); | |
return matches; | |
} | |
(async function() { | |
const body = await request({uri: url}); | |
const matches = getMatched(body, githubRegExp); | |
let projects = []; | |
for (let index in matches) { | |
const projectUrl = matches[index]; | |
let projectBody; | |
try { | |
projectBody = await request({uri: projectUrl}); | |
} catch (err) { | |
console.error(err); | |
continue; | |
} | |
const $ = cheerio.load(projectBody); | |
const stars = $('.js-social-count').text().trim().replace(/,/, ''); | |
if (stars) { | |
try { | |
const prj = { | |
stars: parseInt(stars), | |
url: projectUrl | |
}; | |
projects.push(prj); | |
console.log(`${parseInt(index) + 1} / ${matches.length} [${prj.stars}] ${prj.url}`); | |
} catch (err) { | |
console.error('[ERROR] Wrong stars:', stars, projectUrl); | |
} | |
} | |
} | |
projects = _.sortBy(projects, ['stars']).reverse(); | |
fs.writeFileSync(`./${projectName.split('/')[1]}.json`, JSON.stringify(projects, null, 2)); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment