Pool inflation simulation.

sim.py

from random import randint
from time import sleep


target = .1


def proxy_id_gen():
    c = 1
    while True:
        c += 1
        yield c


px_ids = proxy_id_gen()
spw = 1.  # sec
pool = {1: 0.}
stats = {'avg': target}


def update_stats(i):
    a = 0.9555
    stats['avg'] = a * stats['avg'] + (1. - a) * pool[i]


def policy():
    if target * 0.8 <= stats['avg'] <= target * 1.2:
        return 'noop', 1.
    elif target * 1.1 < stats['avg']:
        return 'inflate', 1.5
    elif stats['avg'] < target * 0.9:
        return 'deflate', 0.8


def inflate(f):
    d = int(round(len(pool) * f) - len(pool))
    assert d >= 0
    for _ in range(0, max(1, d)):
        pool[next(px_ids)] = 0.


def deflate(f):
    d = int(len(pool) - round(len(pool) * f))
    assert d >= 0
    for _ in range(0, d):
        item = pool.popitem()
        # print('D: ', item)


clean_queue = []
clock = 0.
dt = 0.01


def clear_queue():
    px, t = clean_queue[0]
    while t <= clock:
        clean_queue.pop(0)
        if px in pool:
            pool[px] -= 1.
            assert pool[px] >= 0.
            # print('QD: ', px, '-> ', pool[px])
        # else:
        #     print('QDN: ', px)
        px, t = clean_queue[0]


rotation_clock = 0.


def pick_proxy():
    i = randint(0, len(pool) - 1)
    px = list(pool.keys())[i]
    return px


def pool_emptiness():
    e = 0.
    for k in pool:
        if pool[k] == 0:
            e += 1.
    return e / len(pool)


while True:
    px = pick_proxy()
    update_stats(px) # IMPORTANT: this should be before than next
    pool[px] += 1  # Increment queue length. This simulates throttler queue.
    clean_queue.append((px, clock+spw))

    if rotation_clock <= clock:    # If it's time to rotate?
        rotation_clock = clock+2.  # Next time is in 1 sec
        decide, factor = policy()
        if decide == "inflate":
            inflate(factor)
        elif decide == "deflate":
            deflate(factor)
        elif decide == "noop":
            pass
        print(decide, ": ", len(pool), '\t', stats['avg'])
        stats['avg'] = target
    else:
        print('pool', ": ", len(pool), '\t', stats['avg'], '\t', pool_emptiness())

    sleep(dt)
    clock += dt
    clear_queue()