rewriting rsync to the max

gluster_rsync.rb

#!/usr/local/bin/ruby

require 'logger'
require 'optparse'
require 'optparse/time'

require 'thread'
require 'time'
require 'fileutils'

class Server
  attr_accessor :name, :root_path, :files

  def initialize(name, root_path='', files={})
    @name = name
    @root_path = root_path
    @files = files
  end

  def hash
    files.sort.hash
  end

  def eql?(other)
    other.is_a?(self.class) && hash == other.hash
  end

  def self.debug(a, b)
    if a.eql?(b)
      $log.debug "#{a.name} matches #{b.name}"
    else
      $log.debug "#{a.name} does not match #{b.name}"
    end
  end
end

#######################
# POPULATE FILES HASH #
#######################

def grab_files_from_servers
  threads = SERVERS.map do |server|
    Thread.new do
      if server == @client
        grab_files_from_client
      else
        grab_files_from_remote(server)
      end
    end
  end
  threads.each(&:join)
end

def grab_files_from_client
  @client.files.clear if @client_is_src
  directory = File.join(@client.root_path, '**/*')
  files = Dir.glob(directory).reject { |x| File.directory? x }.sort
  files.each do |file|
    name = file.gsub(server.root_path, '')
    date = Time.parse(File.mtime(file).iso8601(5))
    next unless @client.files[name].nil? || @client.files[name] < date
    @semaphore.synchronize { @client.files[name] = date }
  end
end

def grab_files_from_remote(server)
  server.files.clear unless @client_is_src
  script = File.join(server.root_path, 'RSYNC_RUBY_SCRIPT.rb')
  files = `ssh #{server.name} 'export SERVER_PATH="#{server.root_path}" ; #{script}'`
  unless files.nil?
    files = files.split("\n")
    files.each do |file|
      split = file.strip.split(' || ')
      name = split[0].gsub(server.root_path, '')
      date = Time.parse(split[1])
      next unless server.files[name].nil? || server.files[name] < date
      @semaphore.synchronize { server.files[name] = date }
    end
  end
end

###################
# COPY OPERATIONS #
###################

def cp_files_to_volume
  @client.files.each do |file, date|
    source_file = File.join(@client.root_path, file)
    destination_file = File.join(@VOLUME_PATH, file)
    if @brick0.files.include?(file)
      next if date <= @brick0.files[file]
      if @dry_run
        $log.info "COPYING: #{source_file}"
      else
        FileUtils.cp source_file, destination_file, preserve: true, verbose: @verbose
        BRICKS.each { |brick| brick.files[file] = date }
      end
    else
      directory = File.dirname(destination_file)
      if @dry_run
        $log.info "MKDIR: #{directory}"
        $log.info "COPYING: #{source_file}"
      else
        FileUtils.mkdir_p directory, verbose: @verbose
        FileUtils.cp source_file, destination_file, preserve: true, verbose: @verbosr
        BRICKS.each { |brick| brick.files[file] = date }
      end
    end
  end
end

def cp_files_to_standby
  @brick0.files.each do |file, date|
    source_file = File.join(@VOLUME_PATH, file)
    destination_file = File.join(@client.root_path)
    if @client.files.include? file
      next if date <= @client.files[file]
      if @dry_run
        $log.info "COPYING: #{source_file}"
      else
        FileUtils.cp source_file, destination_file, preserve: true, verbose: @verbose
        @client.files[file] = date
      end
    else
      directory = File.dirname(destination_file)
      if @dry_run
        $log.info "MKDIR: #{directory}"
        $log.info "COPYING: #{source_file}"
      else
        FileUtils.mkdir_p directory, verbose: @verbose
        FileUtils.cp source_file, destination_file, preserve: true, verbose: @verbose
        @client.files[file] = date
      end
    end
  end
end

#####################
# REMOVE OPERATIONS #
#####################

def rm_files_from_volume
  rm_files = Array.new
  @brick0.files.each do |file, date|
    next if @client.files.include?(file)
    if @dry_run
      $log.info "REMOVING: #{file}"
    else
      FileUtils.rm File.join(@VOLUME_PATH, file), verbose: @verbose
      BRICKS.each { |brick| brick.files.delete(file) }
    end
  end
end


def rm_files_from_standby
  rm_files = Array.new
  @client.files.each do |file, date|
    next if @brick0.files.include?(file)
    if @dry_run
      $log.info "REMOVING: #{file}"
    else
      FileUtils.rm File.join(@client.root_path, file), verbose: @verbose
      @client.files.delete(file)
    end
  end
end

################
# SANITY CHECK #
################

def compare_all_servers
  SERVERS.map(&:hash).uniq.size == 1
end

def compare_bricks
  BRICKS.map(&:hash).uniq.size == 1
end

###################
# EXECUTE METHODS #
###################

def execute_for_virtual_server
  @client_is_src = true

  synced_paths = [
    { local: '/var/spool/asterisk/voicemail/', volume: "/var/gluster/voicemails/#{ARGV[0]}/" },
    { local: '/var/lib/asterisk/sounds/ogm/', volume: "/var/gluster/greetings/#{ARGV[0]}/" },
    { local: '/var/lib/asterisk/moh/', volume: "/var/gluster/music_on_hold/#{ARGV[0]}/" }
  ]

  synced_paths.each do |hash|
    @VOLUME_PATH = hash[:volume]
    bricks_path = @VOLUME_PATH.gsub('/var/gluster', '/export')

    @client.root_path = hash[:local]
    BRICKS.each { |brick| brick.root_path = bricks_path }

    grab_files_from_servers
    cp_files_to_volume unless compare_all_servers
    rm_files_from_volume if compare_bricks
  end
end

def execute_for_standby_server
  @client_is_src = false

  synced_paths = Hash.new
  VIRTUAL.each do |server|
    synced_paths[server] = [
      { local: "/var/asterisk/voicemails/#{server}/", volume: "/var/gluster/voicemails/#{server}/" },
      { local: "/var/asterisk/ogm/#{server}/", volume: "/var/gluster/greetings/#{server}/" },
      { local: "/var/asterisk/moh/#{server}/", volume: "/var/gluster/music_on_hold/#{server}/" }
    ]
  end

  synced_paths.each do |server, arr|
    arr.each do |hash|
      @VOLUME_PATH = hash[:volume]
      bricks_path = @VOLUME_PATH.gsub('/var/gluster', '/export')

      @client.root_path = hash[:local]
      BRICKS.each { |brick| brick.root_path = bricks_path }

      grab_files_from_servers
      cp_files_to_standby unless compare_all_servers
      rm_files_from_standby if compare_bricks
    end
  end
end

##########################
# CREATE LOG AND OPTIONS #
##########################

$log = Logger.new(STDOUT)
$log.level = Logger::INFO
$log.formatter = proc { |severity, datetime, progname, msg| "[#{datetime} #{severity}] #{msg}\n" }

options = {}

optparse = OptionParser.new do |opts|
  opts.banner = "Usage: ./gluster_rsync.rb [options] LOCAL_HOST\n"
  opts.banner += "  Currently the only supported values for LOCAL_HOST are server0, server1, server2, and standby.\n"

  options[:dry_run] = false
  opts.on('-d', '--dry-run', 'Do *not* copy/remove files. Only print what would have been copied/removed.') do
    options[:dry_run] = true
  end

  options[:verbose] = false
  opts.on('-v', '--verbose', 'Display verbose information on copied/removed files.') do
    options[:verbose] = true
  end

  opts.on( '-h', '--help', 'Display this screen' ) do
    puts opts
    exit
  end
end

optparse.parse!

@dry_run = options[:dry_run]
@verbose = options[:verbose]

###############
# RUN PROGRAM #
###############

@client = Server.new('client')
@brick0 = Server.new('brick0')
@brick1 = Server.new('brick1')
@brick2 = Server.new('brick2')

VIRTUAL = %w( server0 server1 server2 )
SERVERS = [@client, @brick0, @brick1, @brick2]
BRICKS = [@brick0, @brick1, @brick2]

@semaphore = Mutex.new

if ARGV.length == 1
  if VIRTUAL.include? ARGV[0]
    execute_for_virtual_server
  elsif ARGV[0] == 'standby'
    execute_for_standby_server
  else
    puts optparse.banner
    exit
  end
else
  puts optparse.banner
  exit
end

RSYNC_RUBY_SCRIPT.rb

#!/usr/local/bin/ruby

require 'time'

root_path = ENV["SERVER_PATH"]
directory = File.join(root_path, '**/*')
files = Dir.glob(directory).reject{ |x| File.directory? x }.sort
files.each do |file|
  name = file.gsub(root_path, '')
  date = File.mtime(file).iso8601(5)
  print "#{name} || #{date}\n"
end