Print sacred experiments from FileObserver

sacred_exp_print.py

import pandas as pd
import os
import json

def flatten_dict(d):
    """
    transform dict of dicts to single level dict
    """
    fd = {}
    for k, v in d.items():
        if isinstance(v, dict):
            fd_ = flatten_dict(v)
            fd_ = {f'{k}.{k_}': v_ for k_, v_ in fd_.items()}
            fd.update(fd_)
        else:
            fd.update({k: v})
    return fd
        

root_path = '/project/gnn/GNN_tracking/sacred_log/'

experiments_struct = {}
for p in os.listdir(root_path):
    ex_st = {}
    if p.isdigit():
        for p_ in os.listdir(os.path.join(root_path, p)):
            if p_.endswith('.json'):
                with open(os.path.join(root_path, p, p_), 'r') as f:
                    data = json.load(f)
                ex_st.update(flatten_dict(data))
        experiments_struct[p] = ex_st

df = pd.DataFrame.from_dict(experiments_struct, orient='index')
# cols = [c for c in df.columns if not c.lower().startswith('meta')]
# drop columns where identical
for column in df.columns:
    df[column] = df[column].apply(str)  # convert lists to hashable str
df=df[[i for i in df if len(set(df[i]))>1]]
pd.set_option('display.max_rows', len(df))
pd.set_option('display.max_columns', len(df.columns))
# pd.set_option('display.max_colwidth', -1)
df