NodeJS Crawler

var Crawler = require("crawler").Crawler;

var c = new Crawler({
    // here you can define, how many pages you want to do in parallel
    "maxConnections":10,

    // This will be called for each crawled page
    "callback":function(error,result,$) {
        // mark this page as available or not based on the reponse
        console.log(result.statusCode);
    }
});

// Queue all your urls in a loop, they all will be push asynchronously to the crawler job
c.queue("http://www.google.de");
c.queue("http://www.amazon.de");
c.queue("http://www.facebook.de");