compute_best_epochs_david.py

# read davids results

import json
import pandas as pd
import os

def compute_best_epochs_david(directory='./trials', objective="max", folds=8):
    # Get a list of all Excel files in the directory
    files = [file for file in os.listdir(directory) if file.endswith('.json')]

    def format_integer_with_zeros(num):
        return "{:03d}".format(num)

    val_mean = 0.0
    best_fold_score = {}
    for fold in range(0, folds):
        best_fold_score[f"f{fold}"] = {"name": "null", "value": 0.0 if objective=="max" else 1.0}
        for trial in range(0,250):
            with open(f'{directory}/{fold}/trial_{format_integer_with_zeros(trial)}/trial.json') as json_file:
                data = json.load(json_file)
                if objective == "max":
                    try:
                        if data["score"] > best_fold_score[f"f{fold}"]["value"]:
                            best_fold_score[f"f{fold}"]["value"] = data["score"]
                            best_fold_score[f"f{fold}"]["name"] = f'{fold}-trial_{format_integer_with_zeros(trial)}-trial.json'

                            file_path = f'{directory}/best/f{fold}.json'
                            os.makedirs(os.path.dirname(file_path), exist_ok=True)
                            # Save the data as JSON
                            with open(file_path, 'w') as file:
                                json.dump(data, file)
                    except Exception as e:
                        print(f"Exception occured at {fold}-{trial}:\n{e}")
                else:
                    if data["score"] < best_fold_score[f"f{fold}"]["value"]:
                        best_fold_score[f"f{fold}"]["value"] = data["score"]
                        best_fold_score[f"f{fold}"]["name"] = f'{fold}-trial_{format_integer_with_zeros(trial)}-trial.json'

                        file_path = f'{directory}/best/f{fold}.json'
                        os.makedirs(os.path.dirname(file_path), exist_ok=True)
                        # Save the data as JSON
                        with open(file_path, 'w') as file:
                            json.dump(data, file)


            #print(best_fold_score[f"f{fold}"])

        #print(f"fold {fold}: {best_fold_score[f'f{fold}']}")
        print(best_fold_score[f'f{fold}']["name"], best_fold_score[f'f{fold}']["value"])
        val_mean += best_fold_score[f'f{fold}']["value"]

    print("---")
    print(f"NestedCrossEval Mean: {val_mean/folds}")