simple binary encoding and decoding

bin.go

package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"math"
	"reflect"
	"strings"
)

func main() {

	type Foo struct {
		ID       uint16
		Name     string
		IsActive bool
	}

	buf := new(bytes.Buffer)
	enc := NewEncoder(buf)
	enc.Encode(Foo{4, "foo struct", true})
	fmt.Println(buf.Bytes())

	dec := NewDecoder(bytes.NewReader(buf.Bytes()))
	res, err := dec.Decode()
	if err != nil {
		log.Printf("got error decoding: %s", err)
	}
	fmt.Printf("%T, %#v\n", res, res)

}

const (
	bufSize = 512
	intSize = 32 << (^uint(0) >> 63)

	Unknown = 0x10

	Nil = 0x11

	Bool      = 0x20
	BoolTrue  = 0x21
	BoolFalse = 0x22

	Float32 = 0x30
	Float64 = 0x31

	Int   = 0x40
	Int8  = 0x41
	Int16 = 0x42
	Int32 = 0x43
	Int64 = 0x44

	Uint   = 0x50
	Uint8  = 0x51
	Uint16 = 0x52
	Uint32 = 0x53
	Uint64 = 0x54

	String = 0x60
	Bytes  = 0x70

	Array  = 0x80
	Map    = 0x90
	Struct = 0xa0
)

type Decoder struct {
	r   *bufio.Reader
	buf []byte
	off int
}

func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{
		r:   bufio.NewReader(r),
		buf: make([]byte, bufSize),
	}
}

func (d *Decoder) checkRead(n int) {
	_, err := d.r.Read(d.buf[d.off : d.off+n])
	if err != nil {
		if err == io.EOF {
			return
		}
		log.Printf("error reading %d bytes: %s\n", n, err)
	}
}

func (d *Decoder) Decode() (any, error) {
	// Read data into buffer
	_, err := d.r.Read(d.buf)
	if err != nil {
		if err != io.EOF {
			return nil, err
		}
	}
	// Decode value
	return d.readValue(), nil
}

func (d *Decoder) readValue() any {
	d.checkRead(1)
	typ := d.buf[d.off]
	var v any
	switch typ {
	case Nil:
		v = d.read1()
	case Bool:
		_, v = d.read2()
	case Float32:
		_, v = d.read5()
	case Float64:
		_, v = d.read9()
	case Int8:
		_, v = d.read2()
	case Int16:
		_, v = d.read3()
	case Int32:
		_, v = d.read5()
	case Int64:
		_, v = d.read9()
	case Uint8:
		_, v = d.read2()
	case Uint16:
		_, v = d.read3()
	case Uint32:
		_, v = d.read5()
	case Uint64:
		_, v = d.read9()
	case String:
		v = d.readString()
	case Bytes:
		v = d.readBytes()
	case Array:
		v = d.readArray()
	case Map:
		v = d.readMap()
	default:
		v = d.readStruct()
	}
	return v
}

func (d *Decoder) read1() (v uint8) {
	d.checkRead(1)
	v = d.buf[d.off]
	d.off += 1
	return v
}

func (d *Decoder) read2() (t uint8, v uint8) {
	d.checkRead(2)
	t = d.buf[d.off]
	d.off += 1
	v = d.buf[d.off]
	d.off += 1
	return t, v
}

func (d *Decoder) read3() (t uint8, v uint16) {
	d.checkRead(3)
	t = d.buf[d.off]
	d.off += 1
	v = binary.BigEndian.Uint16(d.buf[d.off : d.off+2])
	d.off += 2
	return t, v
}

func (d *Decoder) read5() (t uint8, v uint32) {
	d.checkRead(5)
	t = d.buf[d.off]
	d.off += 1
	v = binary.BigEndian.Uint32(d.buf[d.off : d.off+4])
	d.off += 4
	return t, v
}

func (d *Decoder) read9() (t uint8, v uint64) {
	d.checkRead(9)
	t = d.buf[d.off]
	d.off += 1
	v = binary.BigEndian.Uint64(d.buf[d.off : d.off+8])
	d.off += 8
	return t, v
}

func (d *Decoder) readString() (v string) {
	_, sz := d.read3()
	n := int(sz)
	d.checkRead(n)
	var sb strings.Builder
	sb.Grow(n)
	sb.Write(d.buf[d.off : d.off+n])
	d.off += n
	return sb.String()
}

func (d *Decoder) readBytes() (v []byte) {
	_, sz := d.read5()
	n := int(sz)
	d.checkRead(n)
	v = make([]byte, n)
	copy(v, d.buf[d.off:d.off+n])
	d.off += n
	return v
}

func (d *Decoder) readArray() (v []any) {
	// max elements in array = 4,294,967,295
	_, sz := d.read5()
	n := int(sz)
	v = make([]any, n)
	for i := 0; i < n; i++ {
		v[i] = d.readValue()
	}
	return v
}

func (d *Decoder) readMap() (v map[string]any) {
	// max elements in map = 4,294,967,295
	_, sz := d.read5()
	n := int(sz)
	v = make(map[string]any, n)
	for i := 0; i < n; i++ {
		v[d.readString()] = d.readValue()
	}
	return v
}

func decAlloc(v reflect.Value) reflect.Value {
	for v.Kind() == reflect.Pointer {
		if v.IsNil() {
			v.Set(reflect.New(v.Type().Elem()))
		}
		v = v.Elem()
	}
	return v
}

func (d *Decoder) readStruct() (v any) {
	// struct layout
	// [type][numFields][fieldName][fieldValue]

	// get type and num fields
	typ, num := d.read3()
	if typ != Struct {
		return nil
	}

	// create slice of fields
	fields := make([]reflect.StructField, num)
	values := make([]reflect.Value, num)
	for i := 0; i < int(num); i++ {
		// read field name
		fn := d.readString()

		// read field value
		fv := d.readValue()

		// fill out struct field
		fields[i] = reflect.StructField{
			Name: fn,
			Type: reflect.TypeOf(fv),
		}
		// add to value
		values[i] = reflect.ValueOf(fv)
	}

	// create new struct type
	sct := reflect.New(reflect.StructOf(fields)).Elem()

	// set the values for each field
	for i := 0; i < int(num); i++ {
		sct.Field(i).Set(decAlloc(values[i]))
	}

	// return new struct
	return sct.Addr().Interface()
}

type Encoder struct {
	w   *bufio.Writer
	buf []byte
	off int
}

func NewEncoder(w io.Writer) *Encoder {
	return &Encoder{
		w:   bufio.NewWriter(w),
		buf: make([]byte, bufSize),
	}
}

func (e *Encoder) checkWrite(n int) {
	// check to see if we have room in the current buffer
	if n < len(e.buf[e.off:]) {
		// we have room, so just return
		return
	}
	// check to see if we can fit n bytes in our buffer
	if n < len(e.buf) {
		// looks like we can, but we have to clear it
		// before writing more...
		_, err := e.w.Write(e.buf[:e.off])
		if err != nil {
			panic("error writing buffer")
		}
		// now we can reset it, so our write will work
		e.buf = e.buf[:0]
		e.off = 0
		return
	}
	// check to see if we need to grow our buffer
	if n > cap(e.buf) {
		// looks like we do...
		e.buf = growSlice(e.buf[e.off:], e.off+n)
	}
}

// growSlice grows b by n, preserving the original content of b.
// If the allocation fails, it panics with ErrTooLarge.
//
// This code was ripped end of the go source found at the link below:
// https://cs.opensource.google/go/go/+/master:src/bytes/buffer.go;l=229
func growSlice(b []byte, n int) []byte {
	defer func() {
		if recover() != nil {
			panic(bytes.ErrTooLarge)
		}
	}()
	// TODO(http://golang.org/issue/51462): We should rely on the append-make
	// pattern so that the compiler can call runtime.growslice. For example:
	//	return append(b, make([]byte, n)...)
	// This avoids unnecessary zero-ing of the first len(b) bytes of the
	// allocated slice, but this pattern causes b to escape onto the heap.
	//
	// Instead use the append-make pattern with a nil slice to ensure that
	// we allocate buffers rounded up to the closest size class.
	c := len(b) + n // ensure enough space for n elements
	if c < 2*cap(b) {
		// The growth rate has historically always been 2x. In the future,
		// we could rely purely on append to determine the growth rate.
		c = 2 * cap(b)
	}
	b2 := append([]byte(nil), make([]byte, c)...)
	copy(b2, b)
	return b2[:len(b)]
}

func (e *Encoder) Encode(v any) (err error) {
	e.writeValue(v)
	_, err = e.w.Write(e.buf[:e.off])
	if err != nil {
		return err
	}
	err = e.w.Flush()
	if err != nil {
		return err
	}
	return nil
}

func (e *Encoder) writeValue(v any) {
	switch t := v.(type) {
	case nil:
		e.write1(Nil)
	case bool:
		if t == true {
			e.write2(Bool, BoolTrue)
		}
		e.write2(Bool, BoolFalse)
	case float32:
		e.write5(Float32, math.Float32bits(t))
	case float64:
		e.write9(Float64, math.Float64bits(t))
	case int:
		if intSize == 32 {
			e.write5(Int32, uint32(t))
			break
		}
		e.write9(Int64, uint64(t))
	case int8:
		e.write2(Int8, uint8(t))
	case int16:
		e.write3(Int16, uint16(t))
	case int32:
		e.write5(Int32, uint32(t))
	case int64:
		e.write9(Int64, uint64(t))
	case uint:
		if intSize == 32 {
			e.write5(Uint32, uint32(t))
			break
		}
		e.write9(Uint64, uint64(t))
	case uint8:
		e.write2(Uint8, t)
	case uint16:
		e.write3(Uint16, t)
	case uint32:
		e.write5(Uint32, t)
	case uint64:
		e.write9(Uint64, t)
	case string:
		e.writeString(t)
	case []byte:
		e.writeBytes(t)
	case []any:
		e.writeArray(t)
	case map[string]any:
		e.writeMap(t)
	default:
		val := reflect.Indirect(reflect.ValueOf(v))
		if val.Kind() == reflect.Struct {
			e.writeStruct(val)
		}
		//log.Panicf("Unknown type [%T] and value %v\n", t, v)
	}
}

func (e *Encoder) write1(v uint8) {
	e.checkWrite(1)
	e.buf[e.off] = v
	e.off += 1
}

func (e *Encoder) write2(t uint8, v uint8) {
	e.checkWrite(2)
	e.buf[e.off] = t
	e.off += 1
	e.buf[e.off] = v
	e.off += 1
}

func (e *Encoder) write3(t uint8, v uint16) {
	e.checkWrite(3)
	e.buf[e.off] = t
	e.off += 1
	binary.BigEndian.PutUint16(e.buf[e.off:e.off+2], v)
	e.off += 2
}

func (e *Encoder) write5(t uint8, v uint32) {
	e.checkWrite(5)
	e.buf[e.off] = t
	e.off += 1
	binary.BigEndian.PutUint32(e.buf[e.off:e.off+4], v)
	e.off += 4
}

func (e *Encoder) write9(t uint8, v uint64) {
	e.checkWrite(9)
	e.buf[e.off] = t
	e.off += 1
	binary.BigEndian.PutUint64(e.buf[e.off:e.off+8], v)
	e.off += 8
}

func (e *Encoder) writeString(v string) {
	// max string length = 65,535
	e.write3(String, uint16(len(v)))
	e.checkWrite(len(v))
	n := copy(e.buf[e.off:], v)
	e.off += n
}

func (e *Encoder) writeBytes(v []byte) {
	// max byte slice length = 4,294,967,295
	e.write5(Bytes, uint32(len(v)))
	e.checkWrite(len(v))
	n := copy(e.buf[e.off:], v)
	e.off += n
}

func (e *Encoder) writeArray(v []any) {
	// max elements in array = 4,294,967,295
	e.write5(Array, uint32(len(v)))
	for i := range v {
		e.checkWrite(len(v))
		e.writeValue(v[i])
	}
}

func (e *Encoder) writeMap(v map[string]any) {
	// max elements in map = 4,294,967,295
	e.write5(Map, uint32(len(v)))
	for key, val := range v {
		e.writeString(key)
		e.writeValue(val)
	}
}

func (e *Encoder) writeStruct(v reflect.Value) {
	// struct layout
	// [type][numFields][fieldName][fieldType][fildValue]

	// write type and num fields
	e.write3(Struct, uint16(v.NumField()))
	for i := 0; i < v.NumField(); i++ {
		sf := v.Type().Field(i)
		sv := v.Field(i)

		// write field name
		e.writeString(sf.Name)

		// write field value
		e.writeValue(sv.Interface())
	}
}

func ParseStruct(v any, fn func(reflect.StructField, reflect.Value)) {
	val := reflect.Indirect(reflect.ValueOf(v))
	for i := 0; i < val.NumField(); i++ {
		fn(val.Type().Field(i), val.Field(i))
	}
}