normalize_dict.py

import unicodedata

hankaku_alphabet = "".join(chr(0x21 + i) for i in range(94))
zenkaku_alphabet = "".join(chr(0xff01 + i) for i in range(94))
translate_table = str.maketrans(hankaku_alphabet, zenkaku_alphabet)

original_dict = "/path/to/naist-jdic.csv"
normalized_dict = "/path/to/normalized.csv"

def text_normalize(text: str):
    text = text.translate(translate_table)
    ret = ""
    for s in text:
        if s in zenkaku_alphabet:
            ret += s.lower()
        else:
            ret += unicodedata.normalize("NFC", s)
    return ret

write_log = set()
with open(original_dict, encoding="utf-8") as f:
    with open(normalized_dict, mode="w", encoding="utf-8") as f2:
        for l in f:
            l_splitted = l.split(",")
            l_splitted[0] = text_normalize(l_splitted[0])
            write_contents = ",".join(l_splitted)
            log_contents = ",".join(l_splitted[0:10]+l_splitted[11:])
            if log_contents in write_log:
                continue
            else:
                write_log.add(log_contents)
                f2.write(write_contents)