backup.rake

require 'find'

namespace :db do  desc "Backup the database to a file. Options: DIR=base_dir RAILS_ENV=development MAX=20"
  task backup: [:environment] do
    if Rails.env.production? || ENV['force_backup']
      ENV['backup_base'] = get_backup_base
      backup_file = File.join(get_backup_folder, "#{Rails.env}_dump.sql.gz")

      db_config = ActiveRecord::Base.configurations[Rails.env]

      pass_option = db_config['password'] ? "-p#{db_config['password']}" : ''
      host_option = db_config['host'] ? "-h #{db_config['host']}" : ''

      # Binlogging on dev/staging MySQL servers is not active
      bin_log_option = (Rails.env.development? || Rails.env.staging?) ? "" : "--master-data=2"

      mysqldump_cmd = "mysqldump #{host_option} -u #{db_config['username']} #{pass_option} #{db_config['database']}" +
      " --routines --quote-names --quick --add-drop-table --single-transaction #{bin_log_option}"

      run_mysql_dump!(mysqldump_cmd, backup_file, 100.megabytes)

      Rake::Task["db:remove_old_dumps"].invoke

      upload_to_s3!(backup_file)
    else
      puts "db:backup is available on production only. " +
           "In order to force execution use force_backup option in environment."
    end
  end


  task backup_for_dev: [:environment] do
    if Rails.env.production?|| ENV['force_backup']
      ENV['backup_base'] = get_backup_base('backup_for_dev')
      backup_file = File.join(get_backup_folder, "slave_dump.sql.gz")

      if defined?(Octopus)
        octopus_config = YAML.load(File.open(Rails.root.join("config/shards.yml")))["octopus"]["production"]

        db_config = octopus_config["slave1"]
        fail "Octopus slave1 is not configured for mysql2" unless db_config["adapter"] == "mysql2"
      else
        db_config = ActiveRecord::Base.configurations[Rails.env]
      end

      host = db_config['host'] ? "-h #{db_config['host']}" : ""
      user = "-u #{db_config['username']}"
      pass = db_config['password'] ? "-p#{db_config['password']}" : ""
      database = db_config['database']

      skipped_tables = %w(activities beta_request_metrics beta_requests items products site_metrics short_urls).map{|table| "--ignore-table=#{db_config['database']}.#{table}"}.join(" ")
      skipped_lhma_tables = ActiveRecord::Base.connection.tables.select{|table| /\Alhma_/.match(table) }.map{|table| "--ignore-table=#{db_config['database']}.#{table}"}.join(" ")

      # dump DB structure
      run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} --no-data #{skipped_lhma_tables}", backup_file)
      # dump DB data
      three_months_old = 3.month.ago.strftime("%Y-%m-%d")
      run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} activities split_test_impressions --where='created_at > \"#{three_months_old}\"' --routines --quote-names --quick --add-drop-table --single-transaction", backup_file)
      two_months_old = 2.month.ago.strftime("%Y-%m-%d")
      run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} short_urls --where='updated_at > \"#{two_months_old}\"' --routines --quote-names --quick --add-drop-table --single-transaction", backup_file)
      run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} --routines --quote-names --quick --add-drop-table --single-transaction #{skipped_tables} #{skipped_lhma_tables}", backup_file)

      puts "Finished backup: #{backup_file}"

      ENV["MAX"] = "2"
      Rake::Task["db:remove_old_dumps"].invoke

      upload_to_s3!(backup_file)
    else
      puts "db:backup_for_dev is available on production only. " +
           "In order to force execution use force_backup option in environment."
    end
  end

  task remove_old_dumps: [:environment] do
    max_backups = (ENV["MAX"] || 20).to_i
    all_backups = Dir.new(ENV['backup_base'] || 'db/backup').entries.reject{|d| d.match /^(\.)+(.)*$/}.sort
    puts "Max dumps: #{max_backups}. Current dumps: #{all_backups.count}"
    # reject paths with dot
    unwanted_backups = all_backups.sort.reverse[max_backups..-1] || []
    for unwanted_backup in unwanted_backups
      FileUtils.rm_rf(File.join(ENV['backup_base'], unwanted_backup))
      puts "deleted #{unwanted_backup}"
    end
    puts "Deleted #{unwanted_backups.count} backups, #{all_backups.count - unwanted_backups.count} backups available"
  end

  task "download_s3:dev" => [:environment] do
    s3 = AWS::S3.new(
        access_key_id:     AMAZONS3['access_key_id'],
        secret_access_key: AMAZONS3['secret_access_key']
    )

    bucket = s3.buckets['curebit-backups']

    objs = {}
    bucket.objects.each do |obj|
      objs[obj.key] = obj if obj.key =~ /_slave_dump.sql.gz/
    end

    url = objs.sort.last[1].url_for(:read).to_s

    download_dir = ENV.has_key?("DOWNLOAD_DIR") ? "-d #{ENV["DOWNLOAD_DIR"].shellescape}"  : ""

    sh "aria2c #{download_dir} -x 10 '#{url}'"
  end

  def get_backup_base(dir = nil)
    base_path = ENV["DIR"] || "db"
    ENV['backup_base'] = File.join(base_path, dir || "backup")
  end

  def get_backup_folder
    backup_folder = File.join(ENV['backup_base'], Time.now.strftime("%Y-%m-%d_%H-%M-%S"))
    FileUtils.mkdir_p(backup_folder)
    backup_folder
  end


  def run_mysql_dump!(mysqldump_cmd, backup_file, min_size = 0)
    puts "Starting mysqldump..."

    file_out = File.open(backup_file, mode: "a+b")

    err_r, err_w = IO.pipe
    Open3.pipeline_start(
        [mysqldump_cmd],
        ["gzip -c"],
        out: file_out,  err: err_w) do |ts|
      err_w.close
      error_message = err_r.read # error messages

      fail "Failed to execute mysqldump. #{error_message}" unless ts[0].value.exitstatus == 0
      fail "Failed to gzip mysqldump result. #{error_message}" unless ts[1].value.exitstatus == 0
    end
    file_out.close

    backup_size =  File.size(backup_file)
    fail "Compressed mysqldump result #{backup_file} is too small #{backup_size}. Expected > #{min_size}" unless backup_size > min_size
    puts "Created backup: #{backup_file}"
  end

  def upload_to_s3!(backup_file)
    s3 = AWS::S3.new(
        access_key_id:     AMAZONS3['access_key_id'],
        secret_access_key: AMAZONS3['secret_access_key']
    )

    s3_bucket = s3.buckets['curebit-backups']

    # Folder and file name
    s3_name = "database-backups-last20days/#{File.basename(File.dirname(backup_file))}_#{File.basename(backup_file)}"

    file_obj = s3_bucket.objects[s3_name]
    file_obj.write(file: backup_file)

    puts "Uploaded to S3 as #{s3_name}"
  end

end