wiki scraping

node_api_only.js

var START_PAGE = 'http://ja.wikipedia.org/wiki/Mapion';

function getLinks(url_string, callback) {
  var http = require('http');
  var url = require('url');
  http.get(url.parse(url_string), function(res) {
    var data = [];
    res.on('data', function(chunk) {
      data.push(chunk);
    });
    res.on('end', function() {
      var content = data.join('');
      var result = content.match(/<a[^>]+>/g);
      var links = result.map(function (a) {
        if (a.match(/href="([^"]+)"/)) {
          return RegExp.$1;
        }
      }).filter(function (href) {
        if (!href) return false;
        if (href.indexOf('/wiki/') == 0) return true;
        return false;
      });
      callback(links);
    });
  }).on('error', function(e) {
    console.log('Got error: %s', e.message);
  });
}

getLinks(START_PAGE, function(links) {
  console.log(links.sort());
});

result

$ node node_api_only.js

[ '/wiki/%E3%82%AD%E3%83%A3%E3%83%A9%E3%82%AF%E3%82%BF%E3%83%BC',
  '/wiki/%E3%82%B1%E3%83%BC%E3%82%BF%E3%82%A4%E5%9B%BD%E7%9B%97%E3%82%8A%E5%90%88%E6%88%A6',
  '/wiki/%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF',
  '/wiki/%E3%82%B7%E3%83%A3%E3%83%BC%E3%83%97',
  '/wiki/%E3%82%B9%E3%82%AF%E3%83%AD%E3%83%BC%E3%83%AB',
  '/wiki/%E3%83%8E%E3%83%BC%E3%83%88:Mapion',
  '/wiki/%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB:Computer-blue.svg',
  '/wiki/%E3%83%96%E3%83%AD%E3%83%BC%E3%83%89%E3%83%90%E3%83%B3%E3%83%89',
  '/wiki/%E3%83%97%E3%83%AD%E3%82%B8%E3%82%A7%E3%82%AF%E3%83%88:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF',
  '/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8',
  '/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8',
  '/wiki/%E4%BB%A3%E8%A1%A8%E5%8F%96%E7%B7%A0%E5%BD%B9',
  '/wiki/%E4%BC%9A%E7%A4%BE',
  '/wiki/%E5%87%B8%E7%89%88%E5%8D%B0%E5%88%B7',
  '/wiki/%E5%8D%94%E5%AE%9A%E4%B8%96%E7%95%8C%E6%99%82',
  '/wiki/%E5%9C%B0%E5%9B%B3',
  '/wiki/%E5%A4%A9%E6%B0%97',
  '/wiki/%E6%97%A5%E6%9C%AC',
  '/wiki/%E6%97%A5%E6%9C%AC',
  '/wiki/%E6%9D%B1%E4%BA%AC%E9%83%BD',
  '/wiki/%E6%9D%B1%E6%97%A5%E6%9C%AC%E9%9B%BB%E4%BF%A1%E9%9B%BB%E8%A9%B1',
  '/wiki/%E6%A0%AA%E5%BC%8F%E4%BC%9A%E7%A4%BE',
  '/wiki/%E6%A4%9C%E7%B4%A2',
  '/wiki/%E6%A5%AD%E7%A8%AE',
  '/wiki/%E6%B8%AC%E5%9C%B0%E7%B3%BB',
  '/wiki/%E6%B8%AF%E5%8C%BA_(%E6%9D%B1%E4%BA%AC%E9%83%BD)',
  '/wiki/%E7%89%B9%E5%88%A5:%E3%81%8A%E3%81%BE%E3%81%8B%E3%81%9B%E8%A1%A8%E7%A4%BA',
  '/wiki/%E7%89%B9%E5%88%A5:%E3%82%AB%E3%83%86%E3%82%B4%E3%83%AA',
  '/wiki/%E7%89%B9%E5%88%A5:%E3%83%AA%E3%83%B3%E3%82%AF%E5%85%83/Mapion',
  '/wiki/%E7%89%B9%E5%88%A5:%E5%80%8B%E4%BA%BA%E8%A8%AD%E5%AE%9A',
  '/wiki/%E7%89%B9%E5%88%A5:%E6%96%B0%E3%81%97%E3%81%84%E3%83%9A%E3%83%BC%E3%82%B8',
  '/wiki/%E7%89%B9%E5%88%A5:%E6%9C%80%E8%BF%91%E3%81%AE%E6%9B%B4%E6%96%B0',
  '/wiki/%E7%89%B9%E5%88%A5:%E7%89%B9%E5%88%A5%E3%83%9A%E3%83%BC%E3%82%B8%E4%B8%80%E8%A6%A7',
  '/wiki/%E7%89%B9%E5%88%A5:%E9%96%A2%E9%80%A3%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E6%9B%B4%E6%96%B0%E7%8A%B6%E6%B3%81/Mapion',
  '/wiki/%E7%A4%BE%E9%95%B7',
  '/wiki/%E8%8A%9D%E6%B5%A6',
  '/wiki/%E8%A6%B3%E5%85%89',
  '/wiki/%E9%9B%BB%E9%80%9A',
  '/wiki/1%E6%9C%8810%E6%97%A5',
  '/wiki/1%E6%9C%8815%E6%97%A5',
  '/wiki/1%E6%9C%8820%E6%97%A5',
  '/wiki/1%E6%9C%8820%E6%97%A5',
  '/wiki/10%E6%9C%881%E6%97%A5',
  '/wiki/10%E6%9C%8813%E6%97%A5',
  '/wiki/11%E6%9C%8811%E6%97%A5',
  '/wiki/11%E6%9C%8812%E6%97%A5',
  '/wiki/12%E6%9C%88',
  '/wiki/12%E6%9C%881%E6%97%A5',
  '/wiki/1997%E5%B9%B4',
  '/wiki/1997%E5%B9%B4',
  '/wiki/2%E6%9C%8824%E6%97%A5',
  '/wiki/2002%E5%B9%B4',
  '/wiki/2004%E5%B9%B4',
  '/wiki/2006%E5%B9%B4',
  '/wiki/2007%E5%B9%B4',
  '/wiki/2009%E5%B9%B4',
  '/wiki/4%E6%9C%881%E6%97%A5',
  '/wiki/4%E6%9C%8821%E6%97%A5',
  '/wiki/4%E6%9C%8821%E6%97%A5',
  '/wiki/4%E6%9C%8822%E6%97%A5',
  '/wiki/4%E6%9C%887%E6%97%A5',
  '/wiki/6%E6%9C%881%E6%97%A5',
  '/wiki/7%E6%9C%8830%E6%97%A5',
  '/wiki/8%E6%9C%882%E6%97%A5',
  '/wiki/8%E6%9C%8824%E6%97%A5',
  '/wiki/Ajax',
  '/wiki/Category:%E3%82%AA%E3%83%B3%E3%83%A9%E3%82%A4%E3%83%B3%E6%83%85%E5%A0%B1%E6%BA%90',
  '/wiki/Category:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF%E9%96%A2%E9%80%A3%E3%81%AE%E3%82%B9%E3%82%BF%E3%83%96%E9%A0%85%E7%9B%AE',
  '/wiki/Category:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF%E9%96%A2%E9%80%A3%E3%81%AE%E3%82%B9%E3%82%BF%E3%83%96%E9%A0%85%E7%9B%AE',
  '/wiki/Category:%E5%9C%B0%E5%9B%B3%E4%BC%9A%E7%A4%BE',
  '/wiki/Category:%E5%9C%B0%E7%90%86%E6%8A%80%E8%A1%93',
  '/wiki/Category:%E6%97%A5%E6%9C%AC%E3%81%AE%E5%9C%B0%E5%9B%B3',
  '/wiki/Category:%E6%97%A5%E6%9C%AC%E3%81%AE%E6%83%85%E5%A0%B1%E3%83%BB%E9%80%9A%E4%BF%A1%E6%A5%AD',
  '/wiki/Category:%E6%97%A5%E6%9C%AC%E3%81%AE%E6%83%85%E5%A0%B1%E3%83%BB%E9%80%9A%E4%BF%A1%E6%A5%AD',
  '/wiki/Category:%E6%B8%AF%E5%8C%BA_(%E6%9D%B1%E4%BA%AC%E9%83%BD)%E3%81%AE%E4%BC%81%E6%A5%AD',
  '/wiki/Category:PlaceEngine',
  '/wiki/Google_%E3%83%9E%E3%83%83%E3%83%97',
  '/wiki/Help:%E7%9B%AE%E6%AC%A1',
  '/wiki/Mapion',
  '/wiki/Mapion',
  '/wiki/MapionBB',
  '/wiki/NTT%E3%82%B3%E3%83%9F%E3%83%A5%E3%83%8B%E3%82%B1%E3%83%BC%E3%82%B7%E3%83%A7%E3%83%B3%E3%82%BA',
  '/wiki/Portal:%E3%82%B3%E3%83%B3%E3%83%94%E3%83%A5%E3%83%BC%E3%82%BF',
  '/wiki/Portal:%E6%9C%80%E8%BF%91%E3%81%AE%E5%87%BA%E6%9D%A5%E4%BA%8B',
  '/wiki/Template:%E5%9F%BA%E7%A4%8E%E6%83%85%E5%A0%B1_%E4%BC%9A%E7%A4%BE',
  '/wiki/Web_of_the_Year',
  '/wiki/Wikipedia:%E3%81%8A%E7%9F%A5%E3%82%89%E3%81%9B',
  '/wiki/Wikipedia:%E3%82%A6%E3%82%A3%E3%82%AD%E3%83%9A%E3%83%87%E3%82%A3%E3%82%A2%E3%81%AB%E3%81%A4%E3%81%84%E3%81%A6',
  '/wiki/Wikipedia:%E3%82%B3%E3%83%9F%E3%83%A5%E3%83%8B%E3%83%86%E3%82%A3%E3%83%BB%E3%83%9D%E3%83%BC%E3%82%BF%E3%83%AB',
  '/wiki/Wikipedia:%E3%82%B5%E3%83%B3%E3%83%89%E3%83%9C%E3%83%83%E3%82%AF%E3%82%B9',
  '/wiki/Wikipedia:%E3%82%B9%E3%82%BF%E3%83%96',
  '/wiki/Wikipedia:%E3%83%90%E3%82%B0%E3%81%AE%E5%A0%B1%E5%91%8A',
  '/wiki/Wikipedia:%E3%83%95%E3%82%A1%E3%82%A4%E3%83%AB%E3%81%AE%E3%82%A2%E3%83%83%E3%83%97%E3%83%AD%E3%83%BC%E3%83%89',
  '/wiki/Wikipedia:%E4%BA%95%E6%88%B8%E7%AB%AF',
  '/wiki/Wikipedia:%E5%85%8D%E8%B2%AC%E4%BA%8B%E9%A0%85',
  '/wiki/Wikipedia:%E9%80%A3%E7%B5%A1%E5%85%88',
  '/wiki/Yahoo!%E3%83%9E%E3%83%83%E3%83%97',
  '/wiki/Yahoo!_JAPAN' ]