Instantly share code, notes, and snippets.

Embed
What would you like to do?
AWS S3 read/write Benchmark
source :gemcutter
gem 'pit'
gem 'sauberia-aws-s3'
# s3bench.rb
#
# USAGE:
# $ bundle install
# $ ruby s3bench.rb [N]
#
require "rubygems"
require "benchmark"
require "bundler"
Bundler.setup
require "aws/s3"
require "pit"
include AWS::S3
CONFIG = Pit.get('s3bench', :require => {
'access_key_id' => '',
'secret_access_key' => '',
'bucket_name' => ''
})
BUCKET_NAME = CONFIG['bucket_name']
N = (ARGV.shift || 1).to_i
def dummy_filename
"/tmp/s3bench-dummy"
end
def filename(i)
"s3bench-#{i}"
end
def local_filename(i)
"/tmp/#{filename(i)}"
end
def remote_filename(i)
"s3bench/#{filename(i)}"
end
def bench(title)
puts "\nBenchmark: #{title} (#{N} times)"
Benchmark.bm(7, ">total:", ">avg:") do |bm|
results = []
N.times do |i|
results << bm.report(i.to_s) do
yield i
end
end
total = results.inject {|sum, t| sum+=t}
[total, total/N]
end
end
Base.establish_connection!(
:access_key_id => CONFIG['access_key_id'],
:secret_access_key => CONFIG['secret_access_key'],
:server => "#{BUCKET_NAME}.s3.amazonaws.com"
)
`dd if=/dev/zero of=#{dummy_filename} count=1 bs=5m`
bench("upload") do |i|
S3Object.store(
remote_filename(i),
open(dummy_filename),
BUCKET_NAME
)
end
bench("download") do |i|
open(local_filename(i), 'w') do |file|
S3Object.stream(remote_filename(i), BUCKET_NAME) do |chunk|
file.write chunk
end
end
File.delete local_filename(i)
end
bench("exists?") do |i|
S3Object.exists? remote_filename(i), BUCKET_NAME
end
bench("delete") do |i|
S3Object.delete remote_filename(i), BUCKET_NAME
end
File.delete dummy_filename
@mirakui

This comment has been minimized.

Owner

mirakui commented Jul 19, 2010

From Yahoo! BB mobilepoint to S3(southeast)

Benchmark: upload (10 times)
             user     system      total        real
>total:  0.220000   0.410000   0.630000 (508.744421)
>avg:    0.022000   0.041000   0.063000 ( 50.874442)

Benchmark: download (10 times)
             user     system      total        real
>total:  6.770000   1.250000   8.020000 ( 84.046844)
>avg:    0.677000   0.125000   0.802000 (  8.404684)

Benchmark: exists? (10 times)
             user     system      total        real
>total:  0.020000   0.010000   0.030000 (  3.187673)
>avg:    0.002000   0.001000   0.003000 (  0.318767)

Benchmark: delete (10 times)
             user     system      total        real
>total:  0.010000   0.000000   0.010000 (  2.162521)
>avg:    0.001000   0.000000   0.001000 (  0.216252)
@mirakui

This comment has been minimized.

Owner

mirakui commented Jul 19, 2010

From EC2(southeast) to S3(southeast)

Benchmark: upload (10 times)
             user     system      total        real
>total:  0.000000   0.000000   0.000000 (  3.498857)
>avg:    0.000000   0.000000   0.000000 (  0.349886)

Benchmark: download (10 times)
             user     system      total        real
>total:  0.590000   0.080000   0.670000 (  2.756278)
>avg:    0.059000   0.008000   0.067000 (  0.275628)

Benchmark: exists? (10 times)
             user     system      total        real
>total:  0.010000   0.000000   0.010000 (  0.125816)
>avg:    0.001000   0.000000   0.001000 (  0.012582)

Benchmark: delete (10 times)
             user     system      total        real
>total:  0.000000   0.000000   0.000000 (  0.139655)
>avg:    0.000000   0.000000   0.000000 (  0.013965)
@mirakui

This comment has been minimized.

Owner

mirakui commented Jul 19, 2010

From Flet's Hikari to S3(southeast)

Benchmark: upload (10 times)
             user     system      total        real
>total:  0.230000   0.410000   0.640000 ( 77.704390)
>avg:    0.023000   0.041000   0.064000 (  7.770439)

Benchmark: download (10 times)
             user     system      total        real
>total:  9.350000   1.570000  10.920000 ( 57.733213)
>avg:    0.935000   0.157000   1.092000 (  5.773321)

Benchmark: exists? (10 times)
             user     system      total        real
>total:  0.010000   0.000000   0.010000 (  1.853019)
>avg:    0.001000   0.000000   0.001000 (  0.185302)

Benchmark: delete (10 times)
             user     system      total        real
>total:  0.020000   0.010000   0.030000 (  1.876039)
>avg:    0.002000   0.001000   0.003000 (  0.187604)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment