ec_failure_sim.py

import json
import os
import pprint
import random
import time

from pyeclib import ec_iface

def get_timings(driver, all_slots, repeats, failures, pred=None):
    data = os.urandom(1 << 20)
    frags = driver.encode(data)
    timings = []
    for x in range(repeats):
        available = random.sample(all_slots, len(all_slots) - failures)
        available_frags = [frags[i] for r, i in available if pred is None or pred(r)]
        if not available_frags:
            continue
        try:
            start = time.time()
            decoded = driver.decode(available_frags)
        except ec_iface.ECInsufficientFragments:
            pass
        except ec_iface.ECDriverError:
            pass
        else:
            timings.append(time.time() - start)
            assert decoded == data
    return timings

def failure_sim(spec):
    driver = ec_iface.ECDriver(k=spec['k'], m=spec['m'], ec_type=spec['ec_type'])
    result = {}
    all_slots = [(region, i)
                 for region, indexes in sorted(spec['regions'].items())
                 for i in indexes]

    for failures in spec['failures']:
        timings = get_timings(driver, all_slots, spec['repeats'], failures)
        result[failures] = {
            'can_reconstruct': float(len(timings)) / spec['repeats'],
            #'avg_timing': sum(timings) / len(timings) if timings else None,
            'per_region': {},
        }
        for r in spec['regions']:
            timings = get_timings(driver, all_slots, spec['repeats'], failures,
                                  lambda x: x == r)
            result[failures]['per_region'][r] = {
                'can_reconstruct': float(len(timings)) / spec['repeats'],
                #'avg_timing': sum(timings) / len(timings) if timings else None,
            }
        result[failures]['per_region'] = {
            'can_reconstruct': sum(v['can_reconstruct'] for v in result[failures]['per_region'].values()) / len(result[failures]['per_region']),
        }

    return result


ec_type = 'isa_l_rs_vand'
repeats = 1000
failures =  [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]

scenarios = {
    '2x(8 + 4)': {
        'ec_type': ec_type,
        'k': 8, 'm': 4,
        'regions': {
            '1': range(12),
            '2': range(12),
        },
        'failures': failures,
        'repeats': repeats,
    },
    '8 + 16': {
        'ec_type': ec_type,
        'k': 8, 'm': 16,
        'regions': {
            '1': range(12),
            '2': range(12, 24),
        },
        'failures': failures,
        'repeats': repeats,
    },
    '2x8 + 8': {
        'ec_type': ec_type,
        'k': 8, 'm': 8,
        'regions': {
            '1': range(12),
            '2': range(8) + range(12, 16),
        },
        'failures': failures,
        'repeats': repeats,
    },
    '3x8 + 0': {
        'ec_type': ec_type,
        'k': 8, 'm': 1,  # need one to initialize the driver, but all frags are data
        'regions': {
            '1': range(8) + range(4),
            '2': range(4, 8) + range(8),
        },
        'failures': failures,
        'repeats': repeats,
    },
}

print json.dumps({
    name: failure_sim(scenario)
    for name, scenario in scenarios.items()
})