URL Walker.

urlWalker.js

var _un     = require('underscore');
var url     = require('url');
var util    = require('util');
var request = require('request');

function walkUrl(targetUrl, callback) {
    var probes = {};
    var errBucketIndex = 5;

    // status codes are in the range 100 - 999
    for (var i = 0; i < 10; ++i) {
        probes[i] = [];
    }

    var parsedUrl = url.parse(targetUrl);

    var splitPath = parsedUrl.path.split('/');
    var len = splitPath.length;

    function __walk(i) {
        if (i >= len)
            return callback(probes);

        var target = util.format('%s//%s%s', parsedUrl.protocol, parsedUrl.host, splitPath.slice(0, i+1).join('/'));
        request.get(target, function(_, _, result) {
            var index = errBucketIndex;
            var entry = result;
            try {
                entry = JSON.parse(result);
                index = entry.code/100;
            } catch (ex) {
                entry = {exception: ex, result: result};
            }
            
            // console.log(result);
            probes[index].push({url: target, result: entry});

            __walk(++i);
        });
    };

    return __walk(0);
};

function probeItem(item) {
    console.log(item.url, item.result);
}

function main(argc, argv) {
    var targetUrl = 'https://github.com/odeke-em';
    if (argc >= 3)
        targetUrl = argv[2];

    walkUrl(targetUrl, function(probe) {
        for (var key in probe) {
            _un.map(probe[key] || [], probeItem);
        }
        process.exit(0);
    });
}

var argc = process.argv.length;

if (argc > 1 && process.argv[1] == __filename) {
    main(argc, process.argv);
}