rate_limiter.py

    async def consume_tokens(self):
        consumption_rate = 1 / self.rate_limit
        last_consumption_time = 0
        
        while True:
            if self.tokens_queue.empty():
                await asyncio.sleep(consumption_rate)
                continue

            current_consumption_time = time.monotonic()
            total_tokens = self.tokens_queue.qsize()
            tokens_to_consume = self.get_tokens_amount_to_consume(
                consumption_rate,
                current_consumption_time,
                last_consumption_time,
                total_tokens
            )

            for i in range(0, tokens_to_consume):
                self.token_queue.get_nowait()
                
            last_consumption_time = time.monotonic()

            await asyncio.sleep(consumption_rate)