typhoeus_abort_requests

typhoeus_abort_requests.rb

# frozen_string_literal: true

# Example that demonstrate how to abort requests in hydra when we reach some state.
# It based on :abort feature of streaming response.
#
# See https://github.com/typhoeus/typhoeus#streaming-the-response-body
#
# ```
# $ ABORT_ON_FIRST_REQUEST=false ruby ./typhoeus_abort_requests.rb
# $ ABORT_ON_FIRST_REQUEST=true ruby ./typhoeus_abort_requests.rb
# ```

require 'typhoeus'

class CallbackHandler
  def initialize
    @completed_request = nil
  end

  # Attach callbacks to request
  def call(request)
    request.on_body(&self.method(:on_body))
    request.on_complete(&self.method(:on_complete))
  end

  # Triggered for all requests despite of it state
  def on_body(data, response)
    return :abort if @completed_request
    :unyielded
  end

  # Triggered for all requests despite of it state
  def on_complete(response)
    return if @completed_request
    return unless ENV['ABORT_ON_FIRST_REQUEST'] == 'true'
    @completed_request = response.request.url
  end
end

# Create hydra
hydra = Typhoeus::Hydra.new
# Create handler
callback_handler = CallbackHandler.new

# Requests list
requests = [
  # FAT request (should be processed longer) (1642652 bytes)
  'https://en.wikipedia.org/wiki/Glossary_of_engineering',
  # THIN request (should be processed faster) (1256 bytes)
  'http://www.example.com'
].map do |url|
  Typhoeus::Request.new(url)
end

requests.each do |request|
  # Assign callbacks
  callback_handler.call(request)
  # Add to queue
  hydra.queue(request)
end

t1 = Time.now
# Run hydra reactor (blocking call)
hydra.run
t2 = Time.now

puts "#" * 80
printf "Time: %f\n", t2-t1
puts

# Print results
requests.each do |request|
  puts "#" * 80
  puts "Request  URL: #{request.url}"
  puts "Response success: #{request.response.success?}"
  puts "Response response code: #{request.response.response_code}"
  puts "Response libcurl code: #{request.response.return_code}"
  puts "Response response body size: #{request.response.response_body.size}"
  puts "Response end of response body:\n#{request.response.response_body[-100..-1]}"
  puts
end

There is new upcoming feature typhoeus/ethon#165