very hacky rake semaphore to enable multithreading data migrations in rake tasks

rake_semaphore.rb

namespace :semaphore do
  desc "Basic Semaphore Task"
  task :my_task_name => :environment do
    # Build list of running threads
    @threads = []
    @max_threads = 3
    
    # Setup Rails Config to allow concurrency (prelaoding env for models)
    Rails.configuration.threadsafe!
    Rails.configuration.dependency_loading = true

    model_relation = MyModel.where(thing: 'this')
    puts model_relation


    def thread_work_task(batch)
      @threads << Thread.new do
        batch.each do |item|
          # do my fancy item work with full rails env loaded
          item.sometask if item.persisted? # or whatever
          AnotherModel.where(my_model_id: item.id).each do |another_model|
            another_model.some_method
          end
        end
        @threads.delete Thread.current
      end
    end
    
    def can_proceed?
      if @threads.count >= @max_threads
        puts "Sleeping 3 seconds because threads: #{@threads.count}"
        p @threads
        sleep(3)
        return false
      end
      true
    end

    model_relation.find_in_batches(batch_size: 1000) do |batch|
      # This is where you define how many concurrent threads
      # This count must be lower than your total database conn max
      redo unless can_proceed?
      puts "working batch size #{batch.count}"
      thread_work_task(batch)
    end

    # let's join the last tasks and wait for them to finish
    @threads.each &:join

    puts "FINISHED"
  end
end