gistfile1.rb

def multi_grab(urls,threads=10)
    @hydra = Typhoeus::Hydra.new(:max_concurrency => threads)
    urls.each do |url|
      request = Typhoeus::Request.new(url, :timeout => 10000, :follow_location => true)

      request.on_complete do |response|
        
        yield response

      end
      
      @hydra.queue request
    end

    @hydra.run
    
end

#Grab all urls and stick in array

urls = []

multi_grab(urls,100) do |response|

dead << response.effective_url if response.code == 404

end

# whatevers in dead needs fixing