Skip to content

Instantly share code, notes, and snippets.

@makeding
Last active Jan 13, 2018
Embed
What would you like to do?
let request = require('request');
let cheerio = require('cheerio');
let fs = require('fs');
let tempurls = [];
let headers = {
'pragma': 'no-cache',
'dnt': '1',
'accept-encoding': 'gzip, deflate, br',
'upgrade-insecure-requests': '1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3319.0 Safari/537.36',
'authority': 'www.luogu.org',
'X-Forwarded-For' : '47.89.14.256' //xff
};
request({
url: 'https://www.luogu.org/problem/lists',
headers: headers,
gzip: true
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
let temphtml = cheerio.load(body);
temphtml('.lg-content-table-left a[data-pjax]').map(function(i, el) {
let tempurl = "https://www.luogu.org"+temphtml(this).attr('href');
if(tempurl.indexOf('show')>-1)
tempurls.push(tempurl);
console.log('done');
});
setInterval(function (){
let url = tempurls[0]
tempurls.shift();
request({
url: url,
headers: headers,
gzip: true
}, function (error, response, body) {
if (!error && response.statusCode == 200) {
let html = cheerio.load(body);
console.log((html('title').text()+'\n'+html('div[name=problemleft]').text()+'\nLink: '+url))
fs.writeFileSync('txt/' + url.split('show/')[1] + '.txt',new Buffer(html('title').text()+'\n'+html('div[name=problemleft]').text()+'\nLink: '+url))
}
})
},2000)
}else{
console.error('boom!')
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment