Amazon EC2 Backup

crontab

0 4 * * *  nice -n 19 /var/lib/gems/1.8/bin/rake -f /mnt/backup/Rakefile backup > /mnt/backup/logs/backup.log 2>&1

install.sh

#! /bin/sh

mkdir /mnt/backup
git clone git://gist.github.com/73777.git /mnt/backup/src
ln -s /mnt/backup/src/Rakefile /mnt/backup/Rakefile
cd /mnt/backup
rake init

Rakefile

require 'yaml'
require 'net/smtp'
require 'logger'

DIR         = File.dirname(__FILE__)
KEYS_DIR    = DIR + "/keys"
IMAGE_DIR   = DIR + "/image"
LOGS_DIR    = DIR + "/logs"
CONFIG_FILE = KEYS_DIR + "/config.yml"

logger = Logger.new(ENV["LOG"] || STDOUT)

def permission_check(file)
  if ! File.owned?(file) || 0 != (File.stat(file).mode & 077)
    raise "ERROR: permission: #{file}"
  end
end

def load_config
  if DIR !~ %r{^/mnt(/|$)}
    raise "ERROR: EC2 Backup should be executed in the /mnt/ directory."
  end

  permission_check(KEYS_DIR)
  permission_check(CONFIG_FILE)

  conf = YAML.load_file(CONFIG_FILE)
  conf["pk_file"] = Dir.glob(KEYS_DIR + "/pk-*.pem").first
  conf["cert_file"] = Dir.glob(KEYS_DIR + "/cert-*.pem").first

  permission_check(conf["pk_file"])
  permission_check(conf["cert_file"])

  if conf["ec2_amitool_home"]
    ENV["EC2_AMITOOL_HOME"] = conf["ec2_amitool_home"]
  end

  if conf["cmd_path"]
    conf["cmd_path"] += "/" unless conf["cmd_path"][-1] == ?/
  elsif ENV["EC2_AMITOOL_HOME"] || ENV["EC2_HOME"]
    conf["cmd_path"] = ENV["EC2_AMITOOL_HOME"] || ENV["EC2_HOME"]
    conf["cmd_path"] += (conf["cmd_path"][-1] == ?/ ? "" : "/") + "bin/"
  else
    conf["cmd_path"] = ""
  end

  return conf
end

def exec_cmd(cmd, opts, dryrun = false)
  cmd = [cmd]
  opts.each do |k,v|
    cmd << "--#{k}"
    cmd << v unless v.nil?
  end
  if dryrun
    puts cmd.join(" ")
    return true
  else
    return system(*cmd)
  end
end

def alert(subject, body)
  return unless @conf["alert_to"]
  Net::SMTP.start(@conf["smtp_host"], @conf["smtp_port"]) do |smtp|
    smtp.send_mail <<EOT, @conf["alert_from"], @conf["alert_to"]
From: #{@conf["alert_from"]}
To: #{@conf["alert_to"]}
Subject: #{subject}
Date: #{Time::now.strftime("%a, %d %b %Y %X %z")}

#{body}

S3 bucket: #{@conf["bucket"]}
Image dir: #{IMAGE_DIR}
EOT
  end
end


task :default => :info

desc "Show config."
task :info => :load_config do
  puts "EC2 AMI Tools:   " + %x{#{@conf["cmd_path"]}ec2-ami-tools-version}
  puts "Bucket:          #{@conf["bucket"]}"
  puts "User number:     #{@conf["user_number"]}"
  puts "AWS Access:      #{@conf["access_key"]}"
  puts "AWS Secret:      #{@conf["secret_key"]}"
  puts "X.509 Cert:      #{@conf["cert_file"]}"
  puts "X.509 Private:   #{@conf["pk_file"]}"
  puts "Architecture:    #{@conf["arch"]}"
  puts "SMTP host:       #{@conf["smtp_host"]}:#{@conf["smtp_port"]}"
  puts "Alert mail from: #{@conf["alert_from"]}"
  puts "Alert mail to:   #{@conf["alert_to"]}"
end

desc "Execute backup."
task :backup => [:mkdir, :bundle, :upload, :delete_old, :clean]

task :load_config do
  @conf = load_config
end

task :mkdir => :load_config do
  if File.exist?(IMAGE_DIR)
    alert("[ERROR] EC2 backup failed: Image dir exists.", 
          "Image dir exists. Seemes last backup ended unexpectedly.")
    exit 1
  end
  Dir.mkdir(IMAGE_DIR, 0700) 
end

task :bundle => :load_config do
  logger.info "--- START bundle: #{Time.now.to_s}"
  start = Time.now.to_i
  
  cmd = "#{@conf["cmd_path"]}ec2-bundle-vol"
  opts = {
    :destination => IMAGE_DIR,
    :privatekey  => @conf["pk_file"],
    :cert        => @conf["cert_file"],
    :user        => @conf["user_number"],
    :arch        => @conf["arch"]
  }
  unless exec_cmd(cmd, opts, ENV["DRYRUN"])
    alert("[ERROR] EC2 backup failed: Can't create image.", "See log file.")
    exit 1
  end

  logger.info "--- END bundle: #{Time.now.to_s} (#{Time.now.to_i - start} sec.)"
end

task :upload => :load_config do
  logger.info "--- START upload: #{Time.now.to_s}"
  start = Time.now.to_i

  cmd = "#{@conf["cmd_path"]}ec2-upload-bundle"
  date = Time.now.strftime("%Y%m%d%H%M%S")
  opts = {
    :retry        => nil,
    :bucket       => "#{@conf["bucket"]}/#{date}",
    :manifest     => "#{IMAGE_DIR}/image.manifest.xml",
    :"access-key" => @conf["access_key"],
    :"secret-key" => @conf["secret_key"]
  }
  unless exec_cmd(cmd, opts, ENV["DRYRUN"])
    alert("[ERROR] EC2 backup failed: Can't upload image.", "See log file.")
    exit 1
  end

  logger.info "--- END upload: #{Time.now.to_s} (#{Time.now.to_i - start} sec.)"
end

task :clean => :load_config do
  begin
    FileUtils.rm_r(IMAGE_DIR)
  rescue => e
    alert("[ERROR] EC2 backup failed: Can't remove image dir.", "")
    exit 1
  end
end

def load_s3_objects
  AWS::S3::Base.establish_connection!(
    :access_key_id     => @conf["access_key"],
    :secret_access_key => @conf["secret_key"]
  )
  (bucket, key) = @conf["bucket"].split("/", 2)
  objects = AWS::S3::Bucket.objects(bucket, :prefix => key)
  result = {}
  objects.each do |obj|
    result[obj.key] = obj
  end
  return result
end


task :require_aws_s3 do
  begin
    require 'aws/s3'
  rescue Exception => e
    puts "ERROR: AWS::S3 must be installed."
    puts "       $ sudo gem install aws-s3"
    raise e
  end
end

desc "Show contents of S3"
task :list => [:load_config, :require_aws_s3] do
  objects = load_s3_objects
  objects.keys.sort.each do |k|
    obj = objects[k]
    about = obj.about
    puts obj.key + "\t" + 
         about["last-modified"] + "\t" + 
         about["content-length"]
  end
end

desc "Delete old backups"
task :delete_old => [:load_config, :require_aws_s3] do
  objects = load_s3_objects
  (bucket,prefix) = @conf["bucket"].split("/", 2)
  dates = [] 
  objects.each do |k,v|
    date = k[prefix.length+1, 14]
    next unless date =~ /^\d{14}$/
    dates << date unless dates.include? date
  end
  remove_list = dates.sort.reverse[@conf["hold"].to_i..-1].to_a
  n = 0
  objects.each do |k,v|
    date = k[prefix.length+1, 14]
    if remove_list.include? date
      logger.info "Delete: #{k}"
      v.delete
      n+=1
    end
  end
#  logger.info "Delete #{n} files from #{@conf["bucket"]}/#{remove_list.join(",")}"
end


desc "Try to send alert mail."
task :alert_test => :load_config do
  alert("[TEST] EC2 backup alert test", "This is test.")
end


desc "Initialize environment."
task :init do
  unless File.exist?(KEYS_DIR)
    Dir.mkdir(KEYS_DIR, 0700) 
    logger.info "Created: #{KEYS_DIR}"

    open(KEYS_DIR + "/pk-backup.pem", "w") {} 
    open(KEYS_DIR + "/cert-backup.pem", "w") {}
  end

  unless File.exist?(CONFIG_FILE)
    open(CONFIG_FILE, "w") do |io|
      io << <<EOT
bucket: site.example.com/ami/backup
user_number: 0000-0000-0000
access_key: 0123456789ABCDEFGHIJ
secret_key: 0123456789ABCDEFGHIJ
arch: i386
hold: 3

smtp_host: localhost
smtp_port: 25
alert_from: admin@example.com
alert_to: admin@example.com

# ec2_amitool_home: /usr/local/ec2-ami-tools
# cmd_path: /usr/local/bin
EOT
    end
    File.chmod(0700, CONFIG_FILE)
    logger.info "Created: #{CONFIG_FILE}"
  end

  Dir.mkdir(LOGS_DIR, 0700) unless File.exist?(LOGS_DIR)
  msg = <<EOT 

--------------------------------------------------------------------------------
Initialized.
--------------------------------------------------------------------------------
Edit:
#{CONFIG_FILE}

Put keys:
#{KEYS_DIR}/pk-*.pem
#{KEYS_DIR}/cert-*.pem
--------------------------------------------------------------------------------
EOT
  logger.info msg
end