Site crawler

sitecrawler.js

var phantom = require('phantom');

var Crawler = require("simplecrawler");
var mycrawler = Crawler.crawl("http://www.example.com/");
mycrawler.maxDepth = 3;
mycrawler.interval = 500;
mycrawler.addFetchCondition(function(parsedURL) {
    if (parsedURL.path.match(/\.(css|jpg|pdf|docx|js|png|ico)/i)) {
        // console.log("Ignored ",parsedURL.path);
        return false;
    }

    return true;
});

mycrawler.on("fetchcomplete", function(queueItem) {
    //console.log("Completed fetching resource:", queueItem.url);
    var crawlwait = this.wait();
    phantom.create(function(ph) {
        ph.createPage(function(page) {
            page.open(String(queueItem.url), function(status) {
                console.log("opened " + queueItem.url, status);
                console.log("rendering " + queueItem.path);
              
                page.render("item" + queueItem.path.replace(/\//g, "_") + '.png');
                console.log("finished rendering " + queueItem.path);
              
                crawlwait();
                ph.exit();
            });
        });
    });
});