Fast counters with Redis, persisting to DB

gistfile1.txt

class Counter
  include Sidekiq::Worker
  
  # I'd use a separate Redis instance from Sidekiq
  REDIS = ConnectionPool.new(size: 5) { Redis.new(...) }

  # Call this API in your Rails app code to increment counters
  # as the user does things.
  def self.incr(name, amount=1)
    key = "counter-#{name}-#{current_window}"
    REDIS.with do |conn|
      conn.pipeline do
        conn.incrby(key, amount)
        conn.expire(key, 2.hours)
      end
    end
  end
  
  def self.current_window
    Time.now.strftime("YYYYMMDDHH")
  end
  
  # Use Sidekiq Enterprise's cron feature to run this hourly or
  # whenever each window closes.
  def perform
    REDIS.with do |conn|
      to_ignore = /#{current_window}/
      cursor = "0"
      loop do
        cursor, names = conn.scan(cursor, match: "counter-*", count: 100)
        if names.size > 0
          vals = conn.mget(*names)
          names.zip(vals).each do |counter, value|
            next if counter =~ to_ignore
            next if value == nil # counter expired between scan and mget
            # persist counter with value to PG or use a bulk insert
          end
        end
        break if cursor == "0"
      end
    end
  end
  
end