URL Scraper

scrape.rb

  def scrape_url(url)
    url_found = false
    twitter_name = nil

    begin
      agent = Mechanize.new do |a|
        a.follow_meta_refresh = true
      end

      agent.get(normalize_url(url)) do |page
        url_found = true
        twitter_name = find_twitter_name(page)
      end

      @err << "[#{@current_record}] SUCCESS\n"
    rescue Exception => e
      @err << "[#{@current_record}] ERROR (#{url}): "
      @err << e.message
      @err << "\n"
    end

    [url_found, twitter_name]
  end