refinde.js

var async = require('async'),
    path = require('path'),
    fs = require('fs'),
    $ = require('jquery').create();

function main(){

  var request = require('request');
  var requestURL = "http://tw.news.yahoo.com/sentiment/";
  var mood = new Array();
  var contentURL = "http://tw.news.yahoo.com";
  mood.push("informative/");
  mood.push("warm/");
  mood.push("happy/");
  mood.push("odd/");
  mood.push("boring/");
  mood.push("worried/");
  mood.push("depressing/");
  mood.push("angry/");
  var pages = new Array();
  pages.push(" ");
  pages.push("2.html");
  pages.push("3.html");
  pages.push("4.html");
  pages.push("5.html");


  var max_i = mood.length,
      max_j = 5;
  var doc = [];

  for (var i = 0; i < max_i; i++)
    for (var j = 0; j < max_j; j++)
      doc.push(mood[i] + pages[j]);

  async.timesSeries( max_i * max_j , function(n, callback){

    request(requestURL + doc[n], function (error, response, body) {
      
      if (!error && response.statusCode == 200) {
        var $story = $(body).find(".most-popular-ul .story");
        async.timesSeries( $story.length, function(n2, callback2){
          console.log($story.eq(n2).find("a").attr("href"));
          request(contentURL + $story.eq(n2).find("a").attr("href"), function (error, response, body){
            fs.writeFile("newsCollection/"+( n * $story.length + n2).toString(), $(body).find("#mediaarticlebody .bd").text(), function(err){
              if (err) 
                throw err;
              //console.log('The "data to append" was appended to file!');
              
            });
          callback2();
          });
        },function(err, results){})
      }
      callback();
    });
  },function(err, result){

  });
}
main();