public
Created

  • Download Gist
multiget-server.js
JavaScript
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
/* Multiget web service, written in Node
http://127.0.0.1:9001/?urls=http://example.com/,http://example.net/
Will fetch all URLs in parallel, then return a JSON response containing
the fetched content from them.
*/
 
var DEFAULT_TIMEOUT = 10000;
 
var sys = require("sys"), http = require("http");
 
function multiGet(urls, callback, timeout) {
timeout = timeout || DEFAULT_TIMEOUT;
callback = callback || function() {};
var results = [];
var done = false;
setTimeout(function() {
if (!done) {
// TODO: put error records in the results array
done = true;
callback(results);
}
}, timeout);
urls.forEach(function(url) {
var uri = http.parseUri(url);
if (uri.protocol != 'http') {
results[results.length] = {
'url': url,
'ok': false,
'error': 'Protocol must be HTTP'
};
} else {
var port = parseInt(uri.port || 80, 10);
var client = http.createClient(port, uri.host);
var body = '';
var status = null;
var headers = {}
var request = client.get(uri.path, {"Host": uri.host});
request.finish(function (response) {
sys.puts("... request.finish for " + url);
if (done) {
return; // We've been timed out, ignore it
}
status = response.statusCode;
headers = response.headers;
response.setBodyEncoding("utf8");
response.addListener("body", function(chunk) {
body += chunk;
});
response.addListener("complete", function() {
sys.puts("... response.complete for " + url);
results[results.length] = {
'url': url,
'ok': true,
'headers': headers,
'status': status,
'body': body
};
if (results.length == urls.length) {
done = true;
callback(results);
}
});
});
}
});
if (results.length == urls.length) {
done = true;
callback(results);
}
}
 
http.createServer(function(request, response) {
var urls = (request.uri.params.urls || '').split(',');
multiGet(urls, function(results) {
response.sendHeader(200, {"Content-Type": "text/plain"});
response.sendBody(JSON.stringify({
'results': results
}));
response.finish();
});
}).listen(9001);
 
sys.puts("Server running at http://127.0.0.1:9001/");

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.