EventMachine scraping Hackernews

2_em_hackernews_scraping.rb

require "eventmachine"
require "em-http-request"
require 'nokogiri'

EventMachine.run do

  http_hackernews = EM::HttpRequest.new("https://news.ycombinator.com").get

  http_hackernews.callback do 

    links = Nokogiri::HTML.parse(http_hackernews.response).css("td.title a")

    links.each do |link|
      next unless link
      href = link.attribute("href").to_s

      unless href.match(/http.*/)
        links.delete(link) and next
      end

      http_site = EventMachine::HttpRequest.new(href).get

      http_site.callback do
        links.delete(link)

        begin
          title = Nokogiri.HTML(http_site.response).css("head>title").inner_text
          puts "#{href} - #{http_site.response_header.status} - #{title}"
        rescue
          puts "#{href} - BOOM"
        end

        EventMachine.stop if links.empty?
      end

      http_site.errback do
        puts "#{href} - Error - #{http_site.error.inspect}"
      end

    end
  end
end