concept_dicts.py

python

import json
from pathlib import Path
import copy
import pandas as pd
from datetime import datetime
import numpy as np

json_path = Path(__file__).parent
data_path = Path(__file__).parents[4] / "data" / "picdb" / "V1.1.0"

d_icd_diagnoses = pd.read_csv(data_path / "D_ICD_DIAGNOSES.csv")
d_items = pd.read_csv(data_path / "D_ITEMS.csv")
d_labitems = pd.read_csv(data_path / "D_LABITEMS.csv")

for concept_dict_path in json_path.glob("*.json"):
    with concept_dict_path.open() as f:
        concept_dict = json.load(f)
    
    for concept_name, concept in concept_dict.items():
        
        if concept.get("class") == "rec_cncpt":
            continue

        concept["sources"]["picdb"] = []

        d_items_ = d_items[lambda x: x["LABEL"].str.lower().str.replace("-", "").isin([concept["description"].lower().replace("-", ""), concept_name])]

        if len(d_items_) > 0:
            if not d_items_["UNITNAME"].nunique() == 1:
                breakpoint()
            if isinstance(concept["unit"], str):
                if not (d_items_["UNITNAME"].iloc[0].lower() == concept["unit"].lower()):
                    breakpoint()
            else:
                if not d_items_["UNITNAME"].iloc[0].lower() in [x.lower() for x in concept["unit"]]:
                    breakpoint()
        
            concept["sources"]["picdb"] += d_items_.groupby(["LINKSTO"]).agg({"ITEMID": list}).reset_index().assign(sub_var="itemid").rename(columns={"LINKSTO": "table", "ITEMID": "ids"}).to_dict("records")

        d_labitems_ = d_labitems[lambda x: x["LABEL"].str.lower().str.replace("-", "").isin([concept["description"].lower().replace("-", ""), concept_name])]

        if len(d_labitems_) > 0:
            source_dict = [{"table": "labevents", "ids": d_labitems_["ITEMID"].tolist(), "sub_var": "itemid"}]

            concept["sources"]["picdb"] += source_dict

            # unitnames = labevents[lambda x: x["ITEMID"].isin(d_labitems_["ITEMID"])]["VALUEUOM"].dropna().unique()
            # if not len(unitnames) == 1:
            #     breakpoint()
            # if isinstance(concept["unit"], str):
            #     if not (unitnames[0].lower() == concept["unit"].lower()):
            #         breakpoint()
            # else:
            #     if not unitnames[0].lower() in [x.lower() for x in concept["unit"]]:
            #         breakpoint()
            




        concept_dict[concept_name] = concept
    
    with concept_dict_path.open("w") as f:
        json.dump(concept_dict, f, indent=4)