Multi-threaded HTTP requests that help utilize multiple CPU cores in Rails by requerying the app itself

multi-threaded-http-client-rails.rb

# This way we re-query the app itself in order to avoid sequential execution for CPU-intensive tasks
# that can (and should) be executed in parallel.
#
# Pros: decent scalability can be achieved by assigning path=some.load.balancer.com (i.e. Amazon ELB)
# and putting a bunch of multicore machines behind it.
#
# Cons: we lose some fraction of time on JSON encoding/decoding and HTTP traffic.
# Load-balanced fault-tolerant DRb could be a better choice, but would require more work and provide only
# minor performance improvement in situation where JSON/HTTP aren't the main CPU consumers.

start_date = Date.parse(...)
end_date = Date.parse(...)
params = {...}

day_threads = (start_date..end_date).map { |day|
  path = Rails.application.routes.url_helpers.
    some_resource_path(params.merge(:date => day.to_s))

    req = Net::HTTP::Get.new(path)
    req.basic_auth username, password
    
    http = Net::HTTP.new(host, port)
    http.read_timeout = timeout

  Thread.new {
    ActiveSupport::JSON.decode(http.request(req).body)
  }
}

day_threads.each(&:join)

daily_stats = day_threads.map(&:value)