Created
August 1, 2014 08:02
-
-
Save bogdan/e8b18684914ccf2ec51c to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'find' | |
namespace :db do desc "Backup the database to a file. Options: DIR=base_dir RAILS_ENV=development MAX=20" | |
task backup: [:environment] do | |
if Rails.env.production? || ENV['force_backup'] | |
ENV['backup_base'] = get_backup_base | |
backup_file = File.join(get_backup_folder, "#{Rails.env}_dump.sql.gz") | |
db_config = ActiveRecord::Base.configurations[Rails.env] | |
pass_option = db_config['password'] ? "-p#{db_config['password']}" : '' | |
host_option = db_config['host'] ? "-h #{db_config['host']}" : '' | |
# Binlogging on dev/staging MySQL servers is not active | |
bin_log_option = (Rails.env.development? || Rails.env.staging?) ? "" : "--master-data=2" | |
mysqldump_cmd = "mysqldump #{host_option} -u #{db_config['username']} #{pass_option} #{db_config['database']}" + | |
" --routines --quote-names --quick --add-drop-table --single-transaction #{bin_log_option}" | |
run_mysql_dump!(mysqldump_cmd, backup_file, 100.megabytes) | |
Rake::Task["db:remove_old_dumps"].invoke | |
upload_to_s3!(backup_file) | |
else | |
puts "db:backup is available on production only. " + | |
"In order to force execution use force_backup option in environment." | |
end | |
end | |
task backup_for_dev: [:environment] do | |
if Rails.env.production?|| ENV['force_backup'] | |
ENV['backup_base'] = get_backup_base('backup_for_dev') | |
backup_file = File.join(get_backup_folder, "slave_dump.sql.gz") | |
if defined?(Octopus) | |
octopus_config = YAML.load(File.open(Rails.root.join("config/shards.yml")))["octopus"]["production"] | |
db_config = octopus_config["slave1"] | |
fail "Octopus slave1 is not configured for mysql2" unless db_config["adapter"] == "mysql2" | |
else | |
db_config = ActiveRecord::Base.configurations[Rails.env] | |
end | |
host = db_config['host'] ? "-h #{db_config['host']}" : "" | |
user = "-u #{db_config['username']}" | |
pass = db_config['password'] ? "-p#{db_config['password']}" : "" | |
database = db_config['database'] | |
skipped_tables = %w(activities beta_request_metrics beta_requests items products site_metrics short_urls).map{|table| "--ignore-table=#{db_config['database']}.#{table}"}.join(" ") | |
skipped_lhma_tables = ActiveRecord::Base.connection.tables.select{|table| /\Alhma_/.match(table) }.map{|table| "--ignore-table=#{db_config['database']}.#{table}"}.join(" ") | |
# dump DB structure | |
run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} --no-data #{skipped_lhma_tables}", backup_file) | |
# dump DB data | |
three_months_old = 3.month.ago.strftime("%Y-%m-%d") | |
run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} activities split_test_impressions --where='created_at > \"#{three_months_old}\"' --routines --quote-names --quick --add-drop-table --single-transaction", backup_file) | |
two_months_old = 2.month.ago.strftime("%Y-%m-%d") | |
run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} short_urls --where='updated_at > \"#{two_months_old}\"' --routines --quote-names --quick --add-drop-table --single-transaction", backup_file) | |
run_mysql_dump!("mysqldump #{host} #{user} #{pass} #{database} --routines --quote-names --quick --add-drop-table --single-transaction #{skipped_tables} #{skipped_lhma_tables}", backup_file) | |
puts "Finished backup: #{backup_file}" | |
ENV["MAX"] = "2" | |
Rake::Task["db:remove_old_dumps"].invoke | |
upload_to_s3!(backup_file) | |
else | |
puts "db:backup_for_dev is available on production only. " + | |
"In order to force execution use force_backup option in environment." | |
end | |
end | |
task remove_old_dumps: [:environment] do | |
max_backups = (ENV["MAX"] || 20).to_i | |
all_backups = Dir.new(ENV['backup_base'] || 'db/backup').entries.reject{|d| d.match /^(\.)+(.)*$/}.sort | |
puts "Max dumps: #{max_backups}. Current dumps: #{all_backups.count}" | |
# reject paths with dot | |
unwanted_backups = all_backups.sort.reverse[max_backups..-1] || [] | |
for unwanted_backup in unwanted_backups | |
FileUtils.rm_rf(File.join(ENV['backup_base'], unwanted_backup)) | |
puts "deleted #{unwanted_backup}" | |
end | |
puts "Deleted #{unwanted_backups.count} backups, #{all_backups.count - unwanted_backups.count} backups available" | |
end | |
task "download_s3:dev" => [:environment] do | |
s3 = AWS::S3.new( | |
access_key_id: AMAZONS3['access_key_id'], | |
secret_access_key: AMAZONS3['secret_access_key'] | |
) | |
bucket = s3.buckets['curebit-backups'] | |
objs = {} | |
bucket.objects.each do |obj| | |
objs[obj.key] = obj if obj.key =~ /_slave_dump.sql.gz/ | |
end | |
url = objs.sort.last[1].url_for(:read).to_s | |
download_dir = ENV.has_key?("DOWNLOAD_DIR") ? "-d #{ENV["DOWNLOAD_DIR"].shellescape}" : "" | |
sh "aria2c #{download_dir} -x 10 '#{url}'" | |
end | |
def get_backup_base(dir = nil) | |
base_path = ENV["DIR"] || "db" | |
ENV['backup_base'] = File.join(base_path, dir || "backup") | |
end | |
def get_backup_folder | |
backup_folder = File.join(ENV['backup_base'], Time.now.strftime("%Y-%m-%d_%H-%M-%S")) | |
FileUtils.mkdir_p(backup_folder) | |
backup_folder | |
end | |
def run_mysql_dump!(mysqldump_cmd, backup_file, min_size = 0) | |
puts "Starting mysqldump..." | |
file_out = File.open(backup_file, mode: "a+b") | |
err_r, err_w = IO.pipe | |
Open3.pipeline_start( | |
[mysqldump_cmd], | |
["gzip -c"], | |
out: file_out, err: err_w) do |ts| | |
err_w.close | |
error_message = err_r.read # error messages | |
fail "Failed to execute mysqldump. #{error_message}" unless ts[0].value.exitstatus == 0 | |
fail "Failed to gzip mysqldump result. #{error_message}" unless ts[1].value.exitstatus == 0 | |
end | |
file_out.close | |
backup_size = File.size(backup_file) | |
fail "Compressed mysqldump result #{backup_file} is too small #{backup_size}. Expected > #{min_size}" unless backup_size > min_size | |
puts "Created backup: #{backup_file}" | |
end | |
def upload_to_s3!(backup_file) | |
s3 = AWS::S3.new( | |
access_key_id: AMAZONS3['access_key_id'], | |
secret_access_key: AMAZONS3['secret_access_key'] | |
) | |
s3_bucket = s3.buckets['curebit-backups'] | |
# Folder and file name | |
s3_name = "database-backups-last20days/#{File.basename(File.dirname(backup_file))}_#{File.basename(backup_file)}" | |
file_obj = s3_bucket.objects[s3_name] | |
file_obj.write(file: backup_file) | |
puts "Uploaded to S3 as #{s3_name}" | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment