List of wrong mappings in han-simp.tec

README.md

请见 CTeX-org/ctex-kit#696 (comment)

以 PDF 呈现的列表见 https://drive.google.com/file/d/1zi0UiTisT1UCbowI2esslytDFZq64ONR/view

issue696.csv

main.py

from itertools import chain
from typing import Literal

import numpy as np


class Mapping:
    plane_map: np.ndarray[Literal[16], np.dtype[np.uint8]]
    page_maps: np.ndarray[tuple[Literal[256], Literal[256]], np.dtype[np.uint8]]
    char_maps: np.ndarray[
        tuple[Literal[256], Literal[256], Literal[2]], np.dtype[np.uint32]
    ]

    page_counter: np.uint8
    char_counter: np.uint8

    def __init__(self):
        self.plane_map = np.full(16, 0xFF, dtype=np.uint8)
        self.page_maps = np.full((256, 256), 0xFF, dtype=np.uint8)
        self.char_maps = np.zeros((256, 256, 2), dtype=np.uint32)
        self.page_counter = np.uint8()
        self.char_counter = np.uint8()

    def update(self, ch: np.uint32, mapped: np.uint32):
        """
        https://github.com/silnrsi/teckit/blob/b4a8a2c/source/Compiler.cpp#L3064
        """
        plane = ch >> 16
        assert plane <= 16
        page = (ch >> 8) & 0xFF
        offset = ch & 0xFF

        if self.plane_map[plane] == 0xFF:
            self.plane_map[plane] = self.page_counter
            self.page_counter += 1  # FIXME: overflow
        page_map = self.page_maps[self.plane_map[plane]]

        if page_map[page] == 0xFF:
            page_map[page] = self.char_counter
            self.char_counter += 1  # FIXME: overflow
        char_map = self.char_maps[page_map[page]]

        char_map[offset] = ch, mapped  # FIXME: overwrite

    def lookup(self, ch: np.uint32) -> tuple[np.uint32, np.uint32] | None:
        """
        https://github.com/silnrsi/teckit/blob/b4a8a2c/source/Engine.cpp#L1005
        """
        plane = ch >> 16
        assert plane <= 16
        page = (ch >> 8) & 0xFF
        offset = ch & 0xFF

        if self.plane_map[plane] == 0xFF:
            return None
        page_map = self.page_maps[self.plane_map[plane]]

        if page_map[page] == 0xFF:
            return None
        char_map = self.char_maps[page_map[page]]

        rule_ch, rule_target = char_map[offset]
        if rule_ch == 0:
            return None
        return rule_ch, rule_target


def compile(path: str) -> tuple[Mapping, dict[int, int]]:
    mapping = Mapping()
    reference = {}

    with open(path) as f:
        for line in f.readlines():
            parts = line.split(" <> ")
            if len(parts) != 2:
                continue
            ch = int(parts[0].removeprefix("U+"), 16)
            mapped = int(parts[1].removeprefix("U+"), 16)

            mapping.update(np.uint32(ch), np.uint32(mapped))
            reference[ch] = mapped

    return mapping, reference


def main():
    mapping, reference = compile("han-simp.map")

    wrong: list[tuple[int, int | None, int | None, int | None]] = []

    for ch in chain(range(0x10000), range(0x20000, 0x40000)):
        rule = mapping.lookup(np.uint32(ch))
        reference_mapped = reference.get(ch)
        if rule is None:
            if reference_mapped is not None:
                wrong.append((ch, None, None, reference_mapped))
        else:
            rule_ch, rule_mapped = rule
            if rule_ch != ch:
                wrong.append((ch, int(rule_ch), int(rule_mapped), reference_mapped))

    # print wrong characters in BMP
    print("".join(chr(ch) for ch, _, _, _ in wrong if ch <= 0xFFFF))

    # save wrong characters as CSV
    with open("issue696.csv", "w") as f:
        for ch, rule_ch, rule_mapped, reference_mapped in wrong:
            f.write(f"U+{ch:04X}")
            for c in rule_ch, rule_mapped, reference_mapped:
                if c is None:
                    f.write(", -")
                else:
                    f.write(f", U+{c:04X}")
            f.write("\n")

    # save wrong characters as TeX table
    with open("issue696_table.tex", "w") as f:
        in_tabular = False
        for i, (ch, rule_ch, rule_mapped, reference_mapped) in enumerate(wrong):
            if not in_tabular:
                f.write(
                    r"""\newpage
\begin{tabular}{@{}ABCBCBCBC@{}}
  & \multicolumn{2}{c}{输入字符} & \multicolumn{2}{c}{冲突字符}
  & \multicolumn{2}{c}{错误映射} & \multicolumn{2}{>{\centering}p{6em}@{}}{正确映射} \\
  \hline
"""
                )
                in_tabular = True
            f.write(f"  {i+1}")
            for col, c in enumerate((ch, rule_ch, rule_mapped, reference_mapped)):
                if c is None:
                    if col == 3:
                        f.write(r" & \multicolumn{2}{>{\centering}p{6em}@{}}{无}")
                    else:
                        f.write(r" & \multicolumn{2}{>{\centering}p{6em}}{无}")
                else:
                    if c <= 0xFFFF:
                        f.write(f" & {chr(c)}")
                    else:
                        f.write(rf" & {{\ext {chr(c)}}}")
                    f.write(f" & U+{c:04X}")
            f.write("\\\\\n")
            if (i + 1) % 40 == 0:
                f.write("\\end{tabular}\n")
                in_tabular = False
        if in_tabular:
            f.write("\\end{tabular}\n")


if __name__ == "__main__":
    main()