Skip to content

Instantly share code, notes, and snippets.

@mindreframer
Created January 17, 2013 00:21
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save mindreframer/4552325 to your computer and use it in GitHub Desktop.
Save mindreframer/4552325 to your computer and use it in GitHub Desktop.
S3 downloader (signed URL)
# encoding : utf-8
#!/usr/bin/env ruby
require 'openssl'
require 'digest/sha1'
require 'base64'
require 'date'
require 'uri'
class S3; end
class << S3
def aws_access_key_id
"YOUR_ACCESS_KEY"
end
def aws_secret_access_key
"YOUR_SECRET_ACCESS_KEY"
end
def bucket
"YOUR_BUCKET"
end
end
module Aws
extend self
def signed_url(path, expire_date)
digest = OpenSSL::Digest::Digest.new('sha1')
can_string = "GET\n\n\n#{expire_date}\n/#{S3.bucket}/#{path}"
hmac = OpenSSL::HMAC.digest(digest, S3.aws_secret_access_key, can_string)
signature = URI.escape(Base64.encode64(hmac).strip).encode_signs
"https://#{S3.bucket}.s3.amazonaws.com/#{path}?AWSAccessKeyId=#{S3.aws_access_key_id}&Expires=#{expire_date}&Signature=#{signature}"
end
end
class String
def encode_signs
signs = {'+' => "%2B", '=' => "%3D", '?' => '%3F', '@' => '%40',
'$' => '%24', '&' => '%26', ',' => '%2C', '/' => '%2F', ':' => '%3A',
';' => '%3B', '?' => '%3F'}
signs.keys.each do |key|
self.gsub!(key, signs[key])
end
self
end
end
class Downloader
def pathes
(01..36).to_a.map{|x| "06/backup-20130106.gz.#{x.to_s.rjust(2, '0')}"}
end
def date
@date ||= (Time.now + 10*60*60).to_i # expires in 10 hours
end
def signed_urls
pathes.map do |path|
signed_url = Aws.signed_url(path, date)
end
end
def filename(url)
File.basename(url.split('?').first)
end
def commands
cmds = signed_urls.map do |url|
fname = filename(url)
next if File.exists?(fname) and !File.exists?(fname + ".aria2") # download really finished
#cmd = %Q{wget --no-check-certificate -O=#{filename} "#{url}"}
cmd = %Q{aria2c -s 10 -o #{fname} "#{url}"}
end
(cmds - [nil])
end
def run(should_run=false)
commands.each do |cmd|
if should_run
sh(cmd)
else
puts cmd
end
end
end
# simplified copy of rake`s sh
def sh(cmd)
puts cmd
IO.popen(cmd) do |pipe|
while str = pipe.gets
puts str
end
end
$?.success?
end
end
should_run = ARGV[0] ? true : false
Downloader.new.run(should_run)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment