init_data.py

gistfile1.txt

import sys
import json
from config_initalize import get_cfg_defaults
cfg = get_cfg_defaults()
sys.path.append(cfg.LAB_TOOLS)
print(cfg)
from organize_data import initalize_folders, search_raw_files, Link_origin_to_soft, merge_dicts
initalize_folders(cfg)

raw_files = []
for serach_path in cfg.RAW_SEARCH_TRAIN:
    raw_files.append(search_raw_files(root_path=serach_path, 
                     extension=cfg.RAW_EXTENSION, 
                     include_subfolder=False))
raw_files = [item for sublist in raw_files for item in sublist]

obj = Link_origin_to_soft(path_list=raw_files, 
                          assign_pattern_dict=cfg.TARGET_CLASS, 
                          to_subfolder="train")
obj.build_link(assign_softlink=cfg.OUTPUT.BUILD_SOFTLINK)
train_dict = obj.fetch_dictionary

raw_files = []
for serach_path in cfg.RAW_SEARCH_TEST:
    raw_files.append(search_raw_files(root_path=serach_path, 
                     extension=cfg.RAW_EXTENSION, 
                     include_subfolder=False))
raw_files = [item for sublist in raw_files for item in sublist]

obj = Link_origin_to_soft(path_list=raw_files, 
                          assign_pattern_dict=cfg.TARGET_CLASS, 
                          to_subfolder="test")
obj.build_link(assign_softlink=cfg.OUTPUT.BUILD_SOFTLINK)
test_dict = obj.fetch_dictionary

raw_files = []
for serach_path in cfg.RAW_SEARCH_OTHER:
    raw_files.append(search_raw_files(root_path=serach_path, 
                     extension=cfg.RAW_EXTENSION, 
                     include_subfolder=True))
raw_files = [item for sublist in raw_files for item in sublist]

obj = Link_origin_to_soft(path_list=raw_files, 
                          assign_pattern_dict=cfg.TARGET_CLASS, 
                          to_subfolder="other")
obj.build_link(assign_softlink=cfg.OUTPUT.BUILD_SOFTLINK)
other_dict = obj.fetch_dictionary

if cfg.OUTPUT.WRITE_JSON:
    final_dict = merge_dicts(train_dict, test_dict, other_dict)
    with open("./datalink.json", "w") as f:
        json.dump(final_dict, f)