downloadLinks.js

'use strict';

var http = require('http');
var fs = require('fs');
var Promise = require('bluebird');

var delay = function(time) {
  return new Promise(function(resolve) {
    setTimeout(resolve, time);
  });
};

var streamToBuf = function(stream) {
  return new Promise(function(resolve) {
    var bufs = [];

    stream.on('data', function(d) {
      bufs.push(d);
    });

    stream.on('end', function(){
      resolve(Buffer.concat(bufs));
    });
  });
};

var streamLength = function(stream) {
  return streamToBuf(stream).then(function(buf) {
    return buf.length;
  });
};

var httpGet = function(url) {
  return new Promise(function(resolve, reject) {
    http.get(url, resolve).on('error', reject);
  });
};

var httpGetStats = function(url) {
  var start = new Date();

  return httpGet(url).then(function(response) {
    var end = new Date();

    return streamLength(response).then(function(len) {
      return {
        url: url,
        time: end - start,
        length: len
      }; 
    });
  });
};

var arraySum = function(arr) {
  return arr.reduce(function(a, b) {
    return a + b; 
  });
};

streamToBuf(fs.createReadStream('links.txt')).then(function(buf) {
  var links = buf.toString().split('\n').filter(function(line) {
    return line !== '' && line[0] !== '#';
  });

  Promise.map(
    links,

    function(link) {
      return Promise.race([
        httpGetStats(link),
        delay(1000).then(function() {
          return {
            url: link,
            time: Infinity,
            length: NaN
          }; 
        }) 
      ]);
    }, 

    { concurrency: 5 }
  ).then(function(stats) {

    console.log(stats.sort(function(a, b) {
      return a.time - b.time;
    }).map(function(stats) {
      return (
        stats.url + ', ' +
        (stats.time + 'ms') + ', ' +
        ((stats.length / 1000).toFixed(1) + 'kb')
      ); 
    }).join('\n'));

  });
});

links.txt

#Array.prototype.slice.apply(document.querySelectorAll('.site-listing > div > p > a')).map(function(el) {
#  return el.innerHTML;
#}).join('\n');

http://www.Google.com
http://www.Facebook.com
http://www.Youtube.com
http://www.Yahoo.com
http://www.Baidu.com
http://www.Wikipedia.org
http://www.Amazon.com
http://www.Taobao.com
http://www.Twitter.com
http://www.Qq.com
http://www.Google.co.in
http://www.Live.com
http://www.Sina.com.cn
http://www.Linkedin.com
http://www.Weibo.com
http://www.Yahoo.co.jp
http://www.Tmall.com
http://www.Ebay.com
http://www.Google.co.jp
http://www.Blogspot.com
http://www.Yandex.ru
http://www.Vk.com
http://www.Google.de
http://www.T.co
http://www.Bing.com

http://www.Hao123.com
http://www.Msn.com
http://www.Google.co.uk
http://www.Amazon.co.jp
http://www.Google.fr
http://www.Instagram.com
http://www.Reddit.com
http://www.Google.com.br
http://www.Ask.com
http://www.Sohu.com
http://www.Wordpress.com
http://www.Mail.ru
http://www.Onclickads.net
http://www.Aliexpress.com
http://www.Pinterest.com
http://www.Tumblr.com
http://www.Xvideos.com
http://www.Paypal.com
http://www.Imgur.com
http://www.Google.ru
http://www.Apple.com
http://www.Microsoft.com
http://www.Gmail.com
http://www.Google.it
http://www.Alibaba.com

http://www.Imdb.com
http://www.Fc2.com
http://www.Google.es
http://www.Adcash.com
http://www.360.cn
http://www.Amazon.de
http://www.Diply.com
http://www.Googleadservices.com
http://www.Netflix.com
http://www.Stackoverflow.com
http://www.Craigslist.org
http://www.Google.ca
http://www.Ok.ru
http://www.Xhamster.com
http://www.Naver.com
http://www.Google.com.mx
http://www.Google.com.hk
http://www.Tianya.cn
http://www.Go.com
http://www.Pornhub.com
http://www.Google.com.tr
http://www.Amazon.co.uk
http://www.Kickass.to
http://www.Rakuten.co.jp
http://www.Bbc.co.uk

http://www.Ebay.de
http://www.Amazon.cn
http://www.163.com
http://www.Cnn.com
http://www.Google.pl
http://www.Nicovideo.jp
http://www.Xinhuanet.com
http://www.Dropbox.com
http://www.Dailymotion.com
http://www.Google.co.id
http://www.Google.com.au
http://www.People.com.cn
http://www.Cntv.cn
http://www.Soso.com
http://www.Outbrain.com
http://www.Gmw.cn
http://www.Googleusercontent.com
http://www.Jd.com
http://www.Chinadaily.com.cn
http://www.Bycontext.com
http://www.Youku.com
http://www.Alipay.com
http://www.Flipkart.com
http://www.Adobe.com
http://www.Pixnet.net