Skip to content

Instantly share code, notes, and snippets.

@detorto
Created April 14, 2017 11:12
Show Gist options
  • Save detorto/d60323c3c2fce911effa6cc7c2fa1223 to your computer and use it in GitHub Desktop.
Save detorto/d60323c3c2fce911effa6cc7c2fa1223 to your computer and use it in GitHub Desktop.
Multi-process rate-limiter function. A mutex that unlocks N times per second is used.
import multiprocessing
#used to trigger the permission for request processing
RATED_LOCK = multiprocessing.Lock()
#used for messaging obout the state of operation: can do new, or can't
RATED_QUEUE = multiprocessing.Queue()
#rate-limit decorator, but works only in one thread
def RateLimited(maxPerSecond):
minInterval = 1.0 / float(maxPerSecond)
def decorate(func):
lastTimeCalled = [0.0]
def rateLimitedFunction(*args,**kargs):
elapsed = time.clock() - lastTimeCalled[0]
leftToWait = minInterval - elapsed
if leftToWait>0:
time.sleep(leftToWait)
ret = func(*args,**kargs)
lastTimeCalled[0] = time.clock()
return ret
return rateLimitedFunction
return decorate
def run_ratelimit_dispatcher():
#yes, it is process. we need process not to interfear with main process.
p = multiprocessing.Process(target=rl_dispatch)
p.start()
#this method can be trigerred only N times per second. (per thread/process)
@RateLimited(2.5)
def set_event():
try:
RATED_LOCK.release()
except ValueError:
pass
#to sincronyze set_event beetwin threads/processes we use event queue.
#when there is a message in queue, it means that PERFORM method was invoked and it locked the rated_lock.
#so we can try to unlock it, if rate_limit decorator allows.
def rl_dispatch():
set_event()
while True:
RATED_QUEUE.get()
set_event();
def perform(operation, args):
RATED_LOCK.acquire()
RATED_QUEUE.put("Can unlock")
print ("This will be printed 2.5 times per second")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment