luogu.js

let request = require('request');
let cheerio = require('cheerio');
let fs = require('fs');
let tempurls = [];
let headers = {
    'pragma': 'no-cache',
    'dnt': '1',
    'accept-encoding': 'gzip, deflate, br',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3319.0 Safari/537.36',
    'authority': 'www.luogu.org',
    'X-Forwarded-For' : '47.89.14.256' //xff
};

request({
    url: 'https://www.luogu.org/problem/lists',
    headers: headers,
    gzip: true
}, function (error, response, body) {
    if (!error && response.statusCode == 200) {
        let temphtml = cheerio.load(body);
        temphtml('.lg-content-table-left a[data-pjax]').map(function(i, el) {
            let tempurl = "https://www.luogu.org"+temphtml(this).attr('href');
            if(tempurl.indexOf('show')>-1)
                tempurls.push(tempurl);
            console.log('done');
          });
          setInterval(function (){
                let url = tempurls[0]
                tempurls.shift();
                request({
                    url: url,
                    headers: headers,
                    gzip: true
                }, function (error, response, body) {
                    if (!error && response.statusCode == 200) {
                        let html = cheerio.load(body);
                        console.log((html('title').text()+'\n'+html('div[name=problemleft]').text()+'\nLink: '+url))
                        fs.writeFileSync('txt/' + url.split('show/')[1] + '.txt',new Buffer(html('title').text()+'\n'+html('div[name=problemleft]').text()+'\nLink: '+url))
                    }
                })
            },2000)
    }else{
        console.error('boom!')
    }
});