Spider for ONE

one-spider.js

var Crawler = require("crawler");
var url = require('url');
var fs = require('fs');
var contents = {};
var queue = [];

var c = new Crawler({
    maxConnections : 10,
    forceUTF8: true,
    // This will be called for each crawled page
    callback : function (error, result, $) {
        // $ is Cheerio by default
        //a lean implementation of core jQuery designed specifically for the server
        var content = $('.one-cita-wrapper .one-cita').text().trim();
        var vol = parseInt($('.one-titulo').text().trim().split('.')[1]);
        contents[vol] = content;
        console.log('vol.%s done.', vol);
    },

    onDrain: function() {
        // console.log(contents);
        fs.writeFile('./contents.json', JSON.stringify(contents));
        process.exit();
    }
});


// Queue just one URL, with default callback
// c.queue('http://wufazhuce.com/one/vol.992');

for(var i = 1; i <= 992; i++) {
    queue.push('http://wufazhuce.com/one/vol.' + i);
}

c.queue(queue);