saltperf.py

saltperf.py

#!/usr/bin/env python
'''
calculate from 1 minion to NB_MINIONS the time
to parse the output states dictionnary results

python -m cProfile  -o outfile.stat ./saltperf.py;pyprof2calltree -i outfile.stat -o outfile.pstat

'''

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)

import binascii
import time
from copy import deepcopy
from collections import OrderedDict
import random
import uuid
import string, os
from contextlib import contextmanager

OSAMPLE = OrderedDict([('result', True), ('comment', 'foo')])
NSAMPLE = OrderedDict([('result', False), ('comment', 'bar')])
RESULTS = []
STRINGS = []
NB_MINIONS = 10000  # isnt that much a big number of minions ;)
NB_STATES_PER_MINION = 1000  # already crazy insane to have as much states


@contextmanager
def timeit_context(name):
    startTime = time.time()
    yield
    elapsedTime = time.time() - startTime
    restseconds = ''
    unit = 's'
    if elapsedTime > 60:
        minElapsedTime = elapsedTime / 60.0
        elapsedTime = int(minElapsedTime)
        restseconds = abs(minElapsedTime - elapsedTime) * 60
        unit = 'min'
    RESULTS.append('[{0}] finished in {1}{2}'.format(
        name, elapsedTime, unit, restseconds))


def get_id2():
    # id = str(uuid.uuid4()) #too slow
    id = str(''.join(
        random.sample('azertyuiopsdfghjklmwxcvbn123456789',
                      16)))
    return id


def get_id1():
    # id = str(uuid.uuid4()) #too slow
    chars = string.letters + string.digits
    LEN = 16
    id = ''.join(chars[ord(c) % len(chars)] for c in os.urandom(LEN))
    return id


def get_id():
    id = binascii.b2a_hex(os.urandom(15))
    return id


def populate_ids():
    print('Generating ids')
    nb_ids = max(NB_STATES_PER_MINION, NB_MINIONS) + 1
    seen = []
    for i in range(nb_ids):
        status = int(((100 * i) / nb_ids))
        if (status and (status % 10 == 0)) and not status in seen:
            seen.append(status)
            print("IDS: {0}%".format(status))
        STRINGS.append(get_id())
    print('Generated ids')


def fixture(nb_minions):
    choices = [OSAMPLE, NSAMPLE]
    data = OrderedDict()
    mseen = []
    print("FIXTURE: {0} minions with {1} states each".format(
        nb_minions, NB_STATES_PER_MINION))
    for midx in range(nb_minions):
        minion_id = STRINGS[midx]
        data[minion_id] = OrderedDict()
        seen = []
        for idx in range(NB_STATES_PER_MINION):
            id = STRINGS[idx]
            # only call random on 3 elements, little optim
            if idx % 3:
                choice = random.choice(choices)
            else:
                choice = choices[0]
            data[minion_id][id] = choice
            status = int(((100 * idx) / NB_STATES_PER_MINION))
            #if (status and (status % 99 == 0)) and not status in seen:
            #    seen.append(status)
            #    print("STATES PER MINION: {0}%".format(status))
        status = int(((100 * midx) / nb_minions))
        if (status % 5 == 0) and not status in mseen:
            mseen.append(status)
            print("MINIONS COLLECTION: {0}%".format(status))
    return data


def timed(highstate_data):
    '''Worse case scenario:
        - We parse all the dict
        - We have OrderedDict at all levels
    '''
    foo = True
    for i in highstate_data.keys():
        for j in highstate_data[i].keys():
            if not highstate_data[i][j]['result']:
                foo = False
    return foo


def main():
    populate_ids()
    with timeit_context("INITIALISATION"):
        data = fixture(NB_MINIONS)
        nb_minions = NB_MINIONS
        while(nb_minions >= 1):
            with timeit_context(
                "PROCESSING THE {0} SAMPLES of {1} states".format(
                    nb_minions, len(data[data.keys()[0]]))
            ):
                timed(data)
            nb_minions = int(nb_minions / 10.0)
            data = OrderedDict([(a, data[a])
                                for i, a in enumerate(data)
                                if i <= nb_minions])

        for i in RESULTS:
            print(i)


if __name__ == '__main__':
    main()
#