model runtime evaluation

runtime_evaluation.py

def evaluate_cores(model, data, n_samples, stride=1000, printer=False):
    """
    evaluate multicore model training time
    
    """
    N = [n for n in range(stride, (n_samples+stride), stride)]  # the number of samples we're using, stride of 1000

    cores = os.cpu_count()  # get the number of cores

    model_dict = {f"model_{n}": model(n_jobs=n) for n in range(1, cores+1)}  # contruct model:instance pairs
    data_dict = {model: [] for model in model_dict.keys()}  # prep data for storage

    if printer: print("Starting evaluation...")
    for n in N:
        print(f"\n  n={n}")
        sample = bootstrap(data.copy(), n=n, to_df=True)  # get bootstrap sample data

        for k in data_dict.keys():
            if printer: print(f"\tTraining model {k}")
            start = time.time()
            data_dict[k].append(test_model(model_dict[k], sample))  # add times to dict
            if printer: print(f"\t  - mean fit time: {(time.time() - start):.2f} s")

    data_dict['N'] = N

    df = pd.DataFrame.from_dict(data_dict)

    return df