Skip to content

Instantly share code, notes, and snippets.

@erwanlr
Last active December 14, 2015 14:18
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save erwanlr/5099652 to your computer and use it in GitHub Desktop.
Save erwanlr/5099652 to your computer and use it in GitHub Desktop.
Typhoeus::Hydra issue
source "http://rubygems.org"
gem 'typhoeus', '>=0.6.2'
#gem 'typhoeus', :git => 'git://github.com/typhoeus/typhoeus.git'
gem 'webmock', '>=1.9.3'
gem 'rspec', :require => 'spec'
# spec/spec_helper.rb
require 'webmock/rspec'
require File.expand_path(File.dirname(__FILE__) + '/../test')
require "rubygems"
require "bundler/setup"
require 'typhoeus'
class Foo
attr_reader :target_url
def initialize(target_url)
@target_url = target_url
end
def retrieve_files(max_concurrency = 2)
hydra = Typhoeus::Hydra.new(:max_concurrency => max_concurrency)
found = []
urls.each do |url|
request = Typhoeus::Request.new(url)
request.on_complete do |response|
found << response.request.base_url
end
hydra.queue(request)
end
hydra.run
found
end
def urls
urls = []
Foo.files.each {|file| urls << "#@target_url/#{file}"}
urls
end
def self.files
%w{file1.php file2.php file3.php file4.php}
end
end
# uncomment this for real case
# foo = Foo.new('http://google.com')
# p foo.retrieve_files
# spec/test_spec.rb
require 'spec_helper'
describe Foo do
let(:url) { 'http://google.com' }
subject(:foo) { Foo.new(url) }
before { stub_request(:get, /.*/).to_return(status: 200) }
describe '#retrieve_files' do
(1..4).each do |concurrency|
context "when concurrency = #{concurrency}" do
it 'returns all the files' do
foo.retrieve_files(concurrency).sort.should == foo.urls.sort
end
end
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment