Small web crawler script using Anemone and MongoDB

web_crowler.rb

require 'anemone'
require 'mongo'

# MongoDB setup
db = Mongo::Connection.new.db("demo")
urls_collection = db["page_urls"]

#New Anemone web crawler setup and main operation
Anemone.crawl("http://www.fondazionecollegiopiox.org") do |anemone|
  anemone.storage = Anemone::Storage.MongoDB
  anemone.on_every_page do |page|
    
    puts page.code.to_s + ": " + page.url.to_s

    if page.code && page.url
      url = {code: page.code, url: page.url.to_s}
      puts "Inserting #{url.inspect}"
      urls_collection.insert url
    end
  end
end