Split fcitx5-chinese-addons `sc.dict`

filter.mjs

import { open } from 'node:fs/promises';
import readline from 'node:readline';

const segmenter = new Intl.Segmenter('zh-CN', { granularity: 'grapheme' });

const file = await open('./dict_sc.txt');

const rl = readline.createInterface({
    input: file.createReadStream(),
    crlfDelay: Infinity // \r followed by \n will always be considered a single newline
});

for await (const line of rl) {
    const [hanzi, pinyin, weight] = line.split('\t');
    let hasExtB = false;
    for (const { segment } of segmenter.segment(hanzi)) {
        if (segment.codePointAt(0) > 0x20000) {
            hasExtB = true;
            break;
        }
    }
    if (hasExtB) {
        console.warn(line);
    } else {
        console.log(line);
    }
}

split.sh

#!/bin/sh

curl -L https://download.fcitx-im.org/data/dict_sc.txt-20220628.tar.xz -o dict_sc.tar.xz
tar xf dict_sc.tar.xz
node filter.mjs 1>dict_simp.txt 2>dict_extb.txt