Skip to content

Instantly share code, notes, and snippets.

@bradleypeabody
Last active November 29, 2023 22:08
Show Gist options
  • Star 31 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save bradleypeabody/185b1d7ed6c0c2ab6cec to your computer and use it in GitHub Desktop.
Save bradleypeabody/185b1d7ed6c0c2ab6cec to your computer and use it in GitHub Desktop.
golang, convert UTF-16 to UTF-8 string
package main
// http://play.golang.org/p/fVf7duRtdH
import "fmt"
import "unicode/utf16"
import "unicode/utf8"
import "bytes"
func main() {
b := []byte{
0xff, // BOM
0xfe, // BOM
'T',
0x00,
'E',
0x00,
'S',
0x00,
'T',
0x00,
0x6C,
0x34,
'\n',
0x00,
}
s, err := DecodeUTF16(b)
if err != nil {
panic(err)
}
fmt.Println(s)
}
func DecodeUTF16(b []byte) (string, error) {
if len(b)%2 != 0 {
return "", fmt.Errorf("Must have even length byte slice")
}
u16s := make([]uint16, 1)
ret := &bytes.Buffer{}
b8buf := make([]byte, 4)
lb := len(b)
for i := 0; i < lb; i += 2 {
u16s[0] = uint16(b[i]) + (uint16(b[i+1]) << 8)
r := utf16.Decode(u16s)
n := utf8.EncodeRune(b8buf, r[0])
ret.Write(b8buf[:n])
}
return ret.String(), nil
}
@marians
Copy link

marians commented Sep 28, 2022

This helped me to decode UTF-16LE to UTF-8: https://blog.fearcat.in/a?ID=00001-1bd90844-ce0c-4fac-9b8f-fe3d8a30451d

decoder := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder()
utf8bytes, err := decoder.Bytes(data) // data contains UTF16LE as read from a file

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment