Normalizing Unicode strings in Python / Go / Rust

unicode_normalization.md

Normalizing Unicode strings in Python / Go / Rust

NFC, NFD, NFKC, NFKD

input:

ｉｔ’ｓäå（１−2)ドﾌﾞﾛク㍿

result:

NFC : ｉｔ’ｓäå（１−2)ドﾌﾞﾛク㍿ (45 bytes)
NFD : ｉｔ’ｓäå（１−2)ドﾌﾞﾛク㍿ (50 bytes)
NFKC: it’säå(1−2)ドブロク株式会社 (41 bytes)
NFKD: it’säå(1−2)ドブロク株式会社 (49 bytes)

Python

import unicodedata


def conv_and_print(form):
    src = "ｉｔ’ｓäå（１−2)ドﾌﾞﾛク㍿"
    norm = unicodedata.normalize(form, src)
    print(f"{form}: {norm} ({len(norm.encode('utf-8'))} bytes)")


conv_and_print("NFC")
conv_and_print("NFD")
conv_and_print("NFKC")
conv_and_print("NFKD")

Go

You can use the golang.org/x/text/unicode/norm package.

package main

import (
	"fmt"

	"golang.org/x/text/unicode/norm"
)

func main() {
	src := "ｉｔ’ｓäå（１−2)ドﾌﾞﾛク㍿"
	forms := map[string]norm.Form{"NFC": norm.NFC, "NFD": norm.NFD, "NFKC": norm.NFKC, "NFKD": norm.NFKD}
	for name, form := range forms {
		norm := form.String(src)
		fmt.Printf("%s: %v (%v bytes)\n", name, norm, len(norm))
	}
}

Rust

You can use the unicode-normalization crate.

fn main() {
    use unicode_normalization::UnicodeNormalization;

    let s = "ｉｔ’ｓäå（１−2)ドﾌﾞﾛク㍿";
    let print = |form, norm: &str| println!("{}: {} ({} bytes)", form, norm, norm.len());
    print("NFC", &s.nfc().collect::<String>());
    print("NFD", &s.nfd().collect::<String>());
    print("NFKC", &s.nfkc().collect::<String>());
    print("NFKD", &s.nfkd().collect::<String>());
}