gistfile1.rb

require 'benchmark'
require 'tmpdir'
require 'thread'

AWS.config(access_key_id:    ENV.fetch('AWS_ACCESS_KEY_ID'),
           secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY'))

THREADS_TO_RUN = Integer(ENV["THREADS"] || 2)
store= AWS::S3.new.buckets[ENV.fetch('AWS_BUCKET')].objects

htcat_proc = Proc.new do |dir, name|
  object = store[name]
  url    = object.url_for(:read, expires: 15*60)
  `htcat #{Shellwords.escape(url)} > #{File.join(dir, name)}.tgz`
  raise "failed #{name}" unless $?.success?
end

curl_proc = Proc.new do |dir, name|
  object = store[name]
  url    = object.url_for(:read, expires: 15*60)
  `curl --fail #{Shellwords.escape(url)} -s -o #{dir}/#{name}.tgz`
  raise "failed #{name}" unless $?.success?
end

def run(&block)
  threads = []
  @objects.first(THREADS_TO_RUN).each do |object|
    threads << Thread.new do
      Dir.mktmpdir do |dir|
        block.call(dir, object)
      end
    end
  end
  threads.map(&:join)
end

@objects = [
 # ...
]

Benchmark.bm do |r|
  r.report("curl")    {
    run(&curl_proc)
  }
  r.report("htcat") {
    run(&htcat_proc)
  }
end