HyperLogLog test

hyperloglog_test.go

package main

import (
	"fmt"
	"strconv"

	"github.com/axiomhq/hyperloglog"
	"github.com/c2h5oh/datasize"
)

func estimateError(got, exp uint64) float64 {
	var delta uint64
	if got > exp {
		delta = got - exp
	} else {
		delta = exp - got
	}
	return float64(delta) / float64(exp)
}

func main() {
	hll := hyperloglog.New16()

	step := 10
	unique := map[string]bool{}

	for i := 1; len(unique) <= 10000000; i++ {
		str := "stream-" + strconv.Itoa(i)
		hll.Insert([]byte(str))
		unique[str] = true

		if len(unique)%step == 0 || len(unique) == 10000000 {
			step *= 5
			exact := uint64(len(unique))
			res := hll.Estimate()
			ratio := 100 * estimateError(res, exact)
			data, err := hll.MarshalBinary()
			if err != nil {
				panic(err)
			}
			size := datasize.ByteSize(len(data)).HumanReadable()
			fmt.Printf("Exact %d, got:\n\t HLL %d (%.4f%% off), size: %s\n", exact, res, ratio, size)
		}
	}

}