gistfile1.rb

# This is a way to parallelize long running operations between multiple
# tmux sessions. Basically a poor mans map-reduce for use on caujob02 (or
# other high memory production machines).
#
# See UserAddressUpdate for an example of how to implement something being
# run. Basically you just create a class that inherits from
# RunTmuxSession and supply a query and some code to run for each row
# result of that query. More on this in RunTmuxSession.
#
# This launches {steps} number of tmux sessions and splits the id
# range supplied by the query of the class specified between the tmux
# sessions.
#
# In order be able to start/stop/resume we store a redis key (that expires in
# a week) with a hash of last-id processed. This allows you to kill the tmux
# session at anytime and restart the whole process. If you want to start at
# the beginning just pass in restart:true and it'll nuke the redis hash
#
# You need to run this outside of a tmux session.
#
# When running on production use prodrunner instead of 'be rails r'
#
# Example:
#   # note - there can be no spaces passed in to the string passed to runner
#   # when using prodrunner
#   be rails r "MigrationTools::DistributedTmux::LaunchTmuxSessions.run(
#                 'MigrationTools::DistributedTmux::UserAddressUpdate',10)"
#  optionally you can pass in
#    sleep_delay: seconds to sleep between launching steps
#    restart: false (default) if true it will clear all redis keys
class MigrationTools::DistributedTmux::LaunchTmuxSessions
  class << self
    def run(klass, steps, sleep_delay: 60, restart: false)
      klass = klass.constantize
      steps = steps.to_i

      min_id = klass.query.minimum(:id)
      max_id = klass.query.maximum(:id)
      object_count = max_id - min_id + 1
      step_size = (object_count / steps.to_f).ceil
      session_name = klass.name.demodulize.underscore
      script_name = "#{klass.name}.run(first_id,last_id,step)"

      if restart
        WithRedis.with(klass.redis_hset_name) do |r|
          r.del(klass.redis_hset_name)
        end
      end

      `tmux kill-session -t#{session_name}`

      `tmux new-session -d -s#{session_name} -n0`

      (steps - 1).times do |i|
        `tmux new-window -t#{session_name}:#{i+1} -n#{i+1}`
      end

      batch_count = 0

      runner = if Rails.env.development?
        'bundle exec rails r'
      else
        'prodrunner'
      end

      # TODO: we might want to split up min(:id) max(:id) and send
      # ids instead of doing find_in_batches. Sort of like we do in
      # RunLongUpdate
      steps.times do |i|
        first_id = min_id + (i * step_size)
        last_id  = first_id + step_size - 1

        puts "#{i}: ids: #{first_id} -> #{last_id}"

        exec = %(tmux send-keys -t#{session_name}:#{i}
               '#{runner}
                "#{script_name.gsub('first_id', first_id.to_s)
                              .gsub('last_id', last_id.to_s)
                              .gsub('step', i.to_s)}"'
                C-m).squish
        puts exec
        `#{exec}`

        puts "sleeping for #{sleep_delay}"
        sleep sleep_delay.to_i
      end

      puts "You've created #{steps} tmux windows in session #{session_name}"
      puts `tmux ls`

      puts ""
      puts "You can tmux in via:"
      puts "tmux at -t#{session_name}"
      puts ""
      puts "You can select windows once in tmux via:"
      puts "ctrl-b-w"
      puts ""
      puts "And you can detach back to console:"
      puts "ctrl-b-d"
      puts ""
      puts "And you can kill off this via:"
      puts "tmux kill-session -t#{session_name}"
    end
  end
end