Sose.xyz crawl

DownloadFile.js

var fs = require('fs');
var async = require('async');
var request = require('request');
var url = require('url');
var path = require('path');

var filename = 'thichucolinkko.txt';
var concurrency = 10;

function downloadFile(link, cb) {
    console.log(link);
    var filename = path.basename(url.parse(link).pathname);
    request.head(link, function(err, res, body) {	
        request(link).pipe(fs.createWriteStream(filename)).on('close', cb);
    });
};

function worker(link, cb) {
    console.log('download ' + link);
    downloadFile(link, cb);
}

var queue = async.queue(worker, concurrency);

queue.drain = function() {
    console.log('Done All !');
};

fs.readFile(filename, function(err, data) {
    if (err) throw err;
    var array = data.toString().split("\n");
    for (i in array)
        queue.push(array[i]);
});

GetLink.js

var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var async = require('async');

link = fs.createWriteStream('thichucolinkko.txt');

num_page = 2048;
var concurrency = 30;

var url = 'http://sose.xyz/page/';

function getLinkOnPage(page, cb) {
    console.log('page = ' + page);
    request.get(url + page, function(err, res, body) {
        if (err) throw err;
        var $ = cheerio.load(body);
        var grid = $('div.gridItem.gridphoto');
        grid = grid.toArray();
        grid.forEach(function(img) {
            link.write(img['attribs']['data-photo-high'] + '\n');
        });
        console.log('done page ' + page);
        cb();
    });
}

function worker(page, cb) {
    getLinkOnPage(page, cb);
}

var queue = async.queue(worker, concurrency);

queue.drain = function() {
    console.log('Done All !');
};

for (var i = 0; i <= num_page; i++) {
    queue.push(i);
};