Created
March 10, 2015 10:13
-
-
Save matsadler/efa2d95db19b771bfe0c to your computer and use it in GitHub Desktop.
Simple class to make http requests with multiple connections in separate threads
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require "net/http" | |
# fetcher = FeedCrawler::URLFetcher.new("example.com") | |
# paths = ["/foo", "/bar", "/baz", "/qux"] | |
# results = fetcher.fetch(paths) | |
# results.each do |result| | |
# puts result | |
# end | |
# | |
class URLFetcher | |
class WorkRequest | |
def initialize(input) | |
@input = input | |
@mutex = Mutex.new | |
end | |
def process | |
@mutex.synchronize do | |
@output = yield @input | |
@waiter.wakeup if @waiter | |
@output | |
end | |
end | |
def value | |
@mutex.synchronize do | |
return @output if defined? @output | |
@waiter = Thread.current | |
@mutex.sleep | |
@output | |
end | |
end | |
end | |
def initialize(host, port=80, use_ssl: port == 443, workers: 2) | |
@queue = Queue.new | |
workers.times {start_worker(host, port, use_ssl)} | |
end | |
def fetch(paths) | |
paths.map do |item| | |
request = WorkRequest.new(item) | |
@queue << request | |
request | |
end.each {|req| yield req.value} | |
end | |
private | |
def start_worker(host, port, use_ssl) | |
Thread.new do | |
client = Net::HTTP.new(host, port) | |
client.use_ssl = use_ssl | |
client.start | |
loop {@queue.pop.process {|path| do_request(client, path)}} | |
end.abort_on_exception = true | |
end | |
def do_request(client, path) | |
response = client.get(path) | |
response.value | |
response.body | |
rescue | |
tries = tries.to_i + 1 | |
client.finish | |
client.start | |
retry if tries < 3 | |
raise | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment