http://axe.g0v.tw/level/4 crawer result test in http://jsonlint.com/

axe4.js

var request = require("request");
var cheerio = require("cheerio");

var jar = request.jar();
var headers = {
  "user-agent": "Mozilla/5.0 (Mobile; rv:26.0) Gecko/26.0 Firefox/26.0"
};

var options = {
  jar: jar,
  followRedirect: true,
  maxRedirects: 10,
  headers: headers
};
var last_url;

/**
 * Utility function that downloads a URL and invokes
 * callback with the data.
 */
function download(url, callback) {
  // fill headders
  if (last_url) {
    options.headers.Referer = last_url;
  }
  last_url = url;
  options.url = url;
  request.get(options, function(error, response, body) {
    if (!error && response.statusCode == 200) {
      callback(body);
    }
  });
}


// DO THE JOB
var baseUrl = "http://axe-level-4.herokuapp.com";
var levelUrl = "/lv4/";

var resultJson = [];
var column_title = ["town", "village", "name"];

var queue = [];

request({url: baseUrl}, function () {
  landing();
});

function landing() {
  download(baseUrl + levelUrl, function(data) {
    // console.log(data);
    var $ = cheerio.load(data);
    if (queue.length === 0) {
      $("a").each(function(i, e) {
        var href = $(e).attr("href");
        queue.push(href);
      });
      console.log(queue);
      scheduler();
    }
  });
}

var count=0;
function scheduler() {
  if (count <= queue.length) {
    var path = queue[count];
    count += 1;
    parse(baseUrl + levelUrl + path);
  }
  if (count === queue.length + 1) {
    console.log(JSON.stringify(resultJson));
  }
}

function parse(path) {
  download(path, function(data) {
    console.log(path);
    if (data) {
      var $ = cheerio.load(data);
      $("tr").each(function(i, e) {
        if(i > 0) {
        	var columns = $(e).find("td");
        	// column holder
        	var person = {};
        	columns.each(function(j, e) {
  		      person[column_title[j]] = $(e).text();
        	});
        	resultJson.push(person);
        }
      });
    }
    scheduler();
  });
}