-
-
Save leehambley/6767fbad1f0a66fa90ac to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# | |
# ThreadPool cribbed from: http://burgestrand.se/articles/quick-and-simple-ruby-thread-pool.html | |
# http://burgestrand.se/code/ruby-thread-pool/ | |
# http://burgestrand.se/code/ruby-thread-pool/thread-pool.rb | |
# | |
require 'thread' | |
class ThreadPool | |
def initialize(size) | |
@size = size | |
@jobs = Queue.new | |
@pool = Array.new(@size) do |i| | |
Thread.new do | |
Thread.current[:id] = i | |
catch(:exit) do | |
loop do | |
job, args = @jobs.pop | |
job.call(*args) | |
end | |
end | |
end | |
end | |
end | |
def schedule(*args, &block) | |
@jobs << [block, args] | |
end | |
def shutdown | |
@size.times do | |
schedule { throw :exit } | |
end | |
@pool.map(&:join) | |
end | |
end | |
# | |
# Your Rake Task Body | |
# | |
# | |
namespace :users do | |
task :check_all_the_users do | |
# | |
# Thread Pool will never exceed 150 threads | |
# play with this number to find a good balance | |
# | |
tp = ThreadPool.new(150) | |
Users.find_each(batch_size: 500) do |batch| | |
batch.map do |user| | |
tp.schedule(user) do |user| | |
# Do something with the user here | |
end | |
end | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment