Skip to content

Instantly share code, notes, and snippets.

@dallasmarlow
Created April 23, 2012 21:21
Show Gist options
  • Save dallasmarlow/2473944 to your computer and use it in GitHub Desktop.
Save dallasmarlow/2473944 to your computer and use it in GitHub Desktop.
multipart benchmark
require 'benchmark'
require 's3_multi_upload'
config = {
:access_key_id => 'xxx',
:secret_access_key => 'xxx',
:bucket => :s3_multi_upload,
:file => 'ubuntu-12.04-beta2-dvd-amd64.iso', # 1.6 gb
}
# setup s3 client for serial test
AWS.config :access_key_id => config[:access_key_id],
:secret_access_key => config[:secret_access_key]
s3 = AWS::S3.new
bucket = s3.buckets.create config[:bucket]
Benchmark.bm(10) do |benchmark|
# multipart upload in serial
benchmark.report 'serial' do
100.times do
object = bucket.objects[config[:file]]
object.write :file => config[:file],
:multipart_threshold => 25 * 1024 ** 2 # 25 mb
end
end
# threaded multipart upload
benchmark.report 'parallel' do
100.times do
object = S3_Multi_Upload::Upload.new :access_key_id => config[:access_key_id],
:secret_access_key => config[:secret_access_key],
:bucket => config[:bucket],
:file => config[:file],
:threads => 20,
:chunk_size => {25 => :mb}
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment