Skip to content

Instantly share code, notes, and snippets.

@mcarrowd
Last active January 1, 2024 21:58
Show Gist options
  • Save mcarrowd/e436b6f882897cdbfeb020453dc328f4 to your computer and use it in GitHub Desktop.
Save mcarrowd/e436b6f882897cdbfeb020453dc328f4 to your computer and use it in GitHub Desktop.
golang win1251 <-> utf8 encoding issue demonstration
package main
import (
"fmt"
"golang.org/x/text/encoding/charmap"
)
func main() {
for i := 0; i <= 255; i++ {
charnum := uint8(i)
orig := []uint8{charnum}
utf8 := DecodeWindows1251(orig)
win1251 := EncodeWindows1251(utf8)
if string(orig) != string(win1251) {
fmt.Printf("Oh~.. Failed at charnum 0x%x\n", i)
fmt.Printf("(orig, utf8, win1251)\n")
fmt.Printf("string: %s, %s, %s\n", string(orig), string(utf8), string(win1251))
fmt.Printf("quoted: %+q, %+q, %+q\n", string(orig), string(utf8), string(win1251))
fmt.Printf("hex : % x, % x, % x\n", string(orig), string(utf8), string(win1251))
}
}
}
func DecodeWindows1251(ba []uint8) []uint8 {
dec := charmap.Windows1251.NewDecoder()
out, _ := dec.Bytes(ba)
return out
}
func EncodeWindows1251(ba []uint8) []uint8 {
enc := charmap.Windows1251.NewEncoder()
out, _ := enc.String(string(ba))
return []uint8(out)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment