Parallel

parallel.rb

 def crawl_in_parallel(urls=[], type, subcategory)
    hydra = Typhoeus::Hydra.new(:max_concurrency => 5)

    urls.each do |url|
      request = Typhoeus::Request.new(url, 
        timeout: 7000,
        follow_location: true,
      )
      request.on_complete do |response|
        if response.success?
          begin
            ShopbopProductWrapper.new(url, type, subcategory, response.body)
          rescue Exception => e
            print "F (#{e.message})"
          end
        elsif response.timed_out?
          puts "TIMEOUT for: #{url}"
        elsif response.code == 0
          puts "FAILED HTTP RESPONSE FOR: #{url}"
        else
          puts "HTTP request failed: #{response.code.to_s}"
        end
      end

      hydra.queue request
    end

    hydra.run
  end