compress protobuf

compress.go

package main

import (
	"bytes"
	"compress/flate"
	"compress/gzip"
	"compress/zlib"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/zvelo/adfraud/msg"
)

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func main() {
	var (
		buf     bytes.Buffer
		n       int
		s       time.Time
		elapsed time.Duration
		err     error
		obj     []byte

		test = msg.Request{
			Id: randSeq(1024 * 2),
		}
	)

	t, err := proto.Marshal(&test)
	check(err)
	fmt.Printf("size of protobuf: %d\n\n", len(t))

	buf.Reset()
	fw, err := flate.NewWriter(&buf, 9)
	check(err)
	defer fw.Close()

	s = time.Now()
	n, err = fw.Write(t)
	check(err)
	fw.Close() // need to explicitly close and not just flush to prevent EOF error
	elapsed = time.Since(s)

	obj = buf.Bytes()
	fmt.Printf("written to flated: %d\n", n)
	fmt.Printf("size of flated: %d\n", len(obj))
	fmt.Printf("elapsed flated: %v\n\n", elapsed)
	err = decompressFlate(obj, &test)
	check(err)

	buf.Reset()
	zw := zlib.NewWriter(&buf)
	defer zw.Close()

	s = time.Now()
	n, err = zw.Write(t)
	check(err)
	zw.Flush()
	elapsed = time.Since(s)

	obj = buf.Bytes()
	fmt.Printf("written to zipped: %d\n", n)
	fmt.Printf("size of zlib: %d\n", len(obj))
	fmt.Printf("elapsed zlib: %v\n\n", elapsed)

	buf.Reset()
	gw := gzip.NewWriter(&buf)
	defer gw.Close()

	s = time.Now()
	n, err = gw.Write(t)
	check(err)
	gw.Flush()
	elapsed = time.Since(s)

	obj = buf.Bytes()
	fmt.Printf("written to gzipped: %d\n", n)
	fmt.Printf("size of gzip: %d\n", len(obj))
	fmt.Printf("elapsed gzip: %v\n\n", elapsed)

}

func decompressFlate(obj []byte, expected *msg.Request) error {
	fmt.Println("Decompressing")

	data := bytes.NewReader(obj)
	r := flate.NewReader(data)

	enflate, err := ioutil.ReadAll(r)
	if err != nil {
		panic(err)
	}

	result := msg.Request{}
	proto.Unmarshal(enflate, &result)

	if result.Id != expected.Id {
		fmt.Printf("RESULT[%s] != EXPECTED[%s]\n", result.Id, expected.Id)
		return errors.New("not equal")
	}

	fmt.Println("DECOMPRESSION EQUAL")
	return nil
}

size of protobuf: 2051

written to zipped: 2051
size of zlib: 1510
elapsed zlib: 1.390776ms

written to gzipped: 2051
size of gzip: 1518
elapsed gzip: 1.489147ms

written to flated: 2051
size of flated: 1508
elapsed flated: 685.489µs

size of protobuf: 2051

written to flated: 2051
size of flated: 1508
elapsed flated: 619.604µs

written to zipped: 2051
size of zlib: 1510
elapsed zlib: 1.521965ms

written to gzipped: 2051
size of gzip: 1518
elapsed gzip: 931.558µs

gzip time improves when running after flate