multiget-server.js

/* Multiget web service, written in Node
   
   http://127.0.0.1:9001/?urls=http://example.com/,http://example.net/
   
   Will fetch all URLs in parallel, then return a JSON response containing 
   the fetched content from them.
*/

var DEFAULT_TIMEOUT = 10000;

var sys = require("sys"), http = require("http");

function multiGet(urls, callback, timeout) {
    timeout = timeout || DEFAULT_TIMEOUT;
    callback = callback || function() {};
    var results = [];
    var done = false;
    
    setTimeout(function() {
        if (!done) {
            // TODO: put error records in the results array
            done = true;
            callback(results);
        }
    }, timeout);
    
    urls.forEach(function(url) {
        var uri = http.parseUri(url);
        if (uri.protocol != 'http') {
            results[results.length] = {
                'url': url,
                'ok': false,
                'error': 'Protocol must be HTTP'
            };
        } else {
            var port = parseInt(uri.port || 80, 10);
            var client = http.createClient(port, uri.host);
            var body = '';
            var status = null;
            var headers = {}
            var request = client.get(uri.path, {"Host": uri.host});
            request.finish(function (response) {
                sys.puts("... request.finish for " + url);
                if (done) {
                    return; // We've been timed out, ignore it
                }
                status = response.statusCode;
                headers = response.headers;
                response.setBodyEncoding("utf8");
                response.addListener("body", function(chunk) {
                    body += chunk;
                });
                response.addListener("complete", function() {
                    sys.puts("... response.complete for " + url);
                    results[results.length] = {
                        'url': url,
                        'ok': true,
                        'headers': headers,
                        'status': status,
                        'body': body
                    };
                    if (results.length == urls.length) {
                        done = true;
                        callback(results);
                    }
                });
            });
        }
    });
    if (results.length == urls.length) {
        done = true;
        callback(results);
    }
}

http.createServer(function(request, response) {
    var urls = (request.uri.params.urls || '').split(',');
    multiGet(urls, function(results) {
        response.sendHeader(200, {"Content-Type": "text/plain"});
        response.sendBody(JSON.stringify({
            'results': results
        }));
        response.finish();
    });
}).listen(9001);

sys.puts("Server running at http://127.0.0.1:9001/");