Skip to content

Instantly share code, notes, and snippets.

@ahoward
Created October 4, 2023 20:34
Show Gist options
  • Save ahoward/1945a9765b10e6575c9d18dba68a8861 to your computer and use it in GitHub Desktop.
Save ahoward/1945a9765b10e6575c9d18dba68a8861 to your computer and use it in GitHub Desktop.
# REF: https://platform.openai.com/docs/guides/rate-limits/overview
class RateLimiter < ApplicationRecord
include Tracing
USAGE = -> do
# a multi-process, mutli-thread, multi-machine (assuming shared db), safe rate limiter
# instance level interface
rl = RateLimiter.for(:some, :api, maximum: 1, window: 3)
42.times{ rl.limit{ api.call } }
# class level interface
42.times do
RateLimiter.limit(:some, :api, maximum: 3, window: 7) do
api.call
end
end
# this rate limiter also balances requests at a 'smooth' rate
rl = RateLimiter.for(:foo, :bar, maximum: 3, window: 7)
4.times{ rl.limit{ p :time => Time.now.iso8601(2) } }
<<~____
{:time=>"2023-08-28T01:22:04.15-06:00"}
{:time=>"2023-08-28T01:22:07.98-06:00"}
{:time=>"2023-08-28T01:22:11.49-06:00"}
{:time=>"2023-08-28T01:22:11.50-06:00"}
____
# FUNNY -> https://kracekumar.com/post/chatgpt-gh-profile-lookup/, also, rate limiting is HARD.
# i evaluated the current gems, all buggy under load testing and the demo/repo approach which
# while, simple, would immeadiately break down under MT or MP (if actually deployed as rails app)
# this is a WORK IN PROGRESS
end
DEFAULT = lambda do |attr|
default = {
path: -> { '/rate_limiter' },
count: -> { 0 },
maximum: -> { 3 },
window: -> { 7.0 },
reset_at: -> { Time.now.utc }
}
return default[attr].try(:call)
end
after_initialize :normalize
before_validation :normalize
validates :path, uniqueness: true, allow_nil: false
def self.for(path, *paths, **kws)
path = Path.absolute(path, *paths)
rate_limiter = nil
transaction do
rate_limiter = find_or_create_by!(path:)
rate_limiter.update(**kws) if kws.present?
end
rate_limiter
end
def self.limit(path, *paths, **kws, &block)
RateLimiter.for(path, *paths, **kws).limit(&block)
end
def normalize
self.path = Path.absolute(path || DEFAULT[:path])
self.count ||= DEFAULT[:count]
self.maximum ||= DEFAULT[:maximum]
self.window ||= DEFAULT[:window]
self.last ||= DEFAULT[:last]
self.average ||= DEFAULT[:average]
now = Time.now
self.created_at ||= now
self.updated_at ||= now
self.reset_at ||= now
end
def used
count > 0
end
def stale
elapsed >= window
end
def elapsed
Time.now.utc - reset_at
end
def fucked
(count > maximum) || stale
end
def limit(&block)
seconds = nil
transaction do
if used && (stale || fucked)
trace(
state: 'B',
used:, stale:, fucked:, attributes:
)
update count: DEFAULT[:count], last: DEFAULT[:last], average: DEFAULT[:average], reset_at: DEFAULT[:reset_at]
end
update(count: (count + 1))
if count > 1 && count <= maximum
trace(
state: 'A',
attributes:
)
if last || average
duration = [last, average].compact.max
slow = (maximum * duration) > window
trace(
state: 'A.1',
duration:, slow:
)
unless slow
remaining_time = window - elapsed
remaining_count = maximum - count + 1
trace(
state: 'A.2',
remaining_time:, remaining_count:
)
if remaining_time > 0 && remaining_count > 0
pace = remaining_time / remaining_count
slop = rand(0.0420..0.420)
seconds = (pace - duration) + slop
trace(
state: 'A.3',
pace:, seconds:
)
end
end
end
end
end
if seconds
seconds = [seconds, window].min
trace(
sleep: seconds
)
sleep([seconds, 0].max)
end
time(&block)
end
def time(&block)
timing = []
timing.push Time.now
begin
block.call
ensure
timing.push Time.now
t = timing.last - timing.first
a = average || t
n = count
a -= (a / n)
a += (t / n)
update last: t, average: a
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment