Example of how we may use queue with threads to make efficient background worker.

worker_and_queue_perfect_pair.rb

class Worker
  def self.start(num_threads:, queue_size:)
    queue = SizedQueue.new(queue_size)
    worker = new(num_threads: num_threads, queue: queue)
    worker.spawn_threads
    worker
  end

  def initialize(num_threads:, queue:)
    @num_threads = num_threads
    @queue = queue
    @threads = []
  end
  
  attr_reader :num_threads, :threads
  private :threads
  
  def spawn_threads
    num_threads.times do
      threads << Thread.new do
        while running? || actions?
          action_proc, action_payload = wait_for_action
          action_proc.call(action_payload) if action_proc
        end
      end
    end
  end
  
  def enqueue(action, payload)
    queue.push([action, payload])
  end
  
  def stop
    queue.close
    threads.each(&:exit)
    threads.clear
    true
  end

  private

  attr_reader :queue, :threads

  def actions?
    !queue.empty?
  end
  
  def running?
    !queue.closed?
  end
  
  def dequeue_action
    queue.pop(true)
  end
  
  def wait_for_action
    queue.pop(false)
  end
end

Hi @iamdidev, dequeue_action method is declared as private here. I do not see the usage of this method inside this class. Could you please explain the purpose of this method?

Hi @SathiyarajS good catch. This method is used in the "production" version of the worker, take a look https://github.com/yabeda-rb/yabeda-datadog/blob/8c6e40f24038d4420be500c97032c38096f541e2/lib/yabeda/datadog/worker.rb