zipkin ES response percentiles

main.go

package main

import (
	"flag"
	"fmt"
	"os"
	"reflect"
	"time"

	"github.com/codahale/hdrhistogram"
	"github.com/gonum/plot"
	"github.com/gonum/plot/plotter"
	"github.com/gonum/plot/plotutil"
	"github.com/gonum/plot/vg"
	elastic "gopkg.in/olivere/elastic.v3"
)

var (
	system        string
	client        *elastic.Client
	q             elastic.Query
	generateCSV   bool
	generateImage bool
	percentiles   = []float64{
		10.0,
		20.0,
		30.0,
		40.0,
		50.0,
		55.0,
		60.0,
		65.0,
		70.0,
		75.0,
		77.5,
		80.0,
		82.5,
		85.0,
		87.5,
		88.75,
		90.0,
		91.25,
		92.5,
		93.75,
		94.375,
		95.0,
		95.625,
		96.25,
		96.875,
		97.1875,
		97.5,
		97.8125,
		98.125,
		98.4375,
		98.5938,
		98.75,
		98.9062,
		99.0625,
		99.2188,
		99.2969,
		99.3750,
		99.4531,
		99.5313,
		99.6094,
		99.6484,
		99.6875,
		99.7266,
		99.7656,
		99.8047,
		99.8242,
		99.8437,
		99.8633,
		99.8828,
		99.9023,
		99.9121,
		99.9219,
		99.9316,
		99.9414,
		99.9512,
		99.9561,
		99.9609,
		99.9658,
		99.9707,
		99.9756,
		99.9780,
		99.9805,
		99.9829,
		99.9854,
		99.9878,
		99.9890,
		99.9902,
	}
)

func init() {
	flag.BoolVar(&generateCSV, "csv", true, "generate csv data")
	flag.BoolVar(&generateImage, "img", false, "generate image data")
	flag.Parse()

	system = flag.Args()[0]

	b := elastic.NewBoolQuery()
	b.Must(elastic.NewMatchQuery("binaryAnnotations.endpoint.serviceName", system))

	q = elastic.NewNestedQuery(
		"binaryAnnotations",
		b,
	)

	c, err := elastic.NewClient(
		elastic.SetURL("127.0.0.1:9200"),
		elastic.SetHealthcheck(false),
		elastic.SetSniff(false),
	)
	if c == nil || err != nil {
		panic(fmt.Errorf("Failed retrieving an client: %#v", err))
	}
	client = c
}

func main() {
	fromDate, err := time.Parse("2006-01-02", flag.Args()[1])
	if err != nil {
		panic(err)
	}
	toDate, err := time.Parse("2006-01-02", flag.Args()[2])
	if err != nil {
		panic(err)
	}

	date := fromDate
	if generateCSV {
		fmt.Printf("%q;%q;%q\n", "date", "percentile", "response time in ms")
	}
	for date.Before(toDate) {
		hdr, err := histogram(date)
		if err == nil {
			if generateCSV {
				fmt.Printf("%q;;\n", date.Format("2006-01-02"))
				printCSV(date, hdr)
			}
			if generateImage {
				printImage(date, hdr)
			}
		}
		date = date.Add(time.Duration(24) * time.Hour)
	}

	hdr, _ := histogram(toDate)
	if generateCSV {
		fmt.Printf("%q;;\n", toDate.Format("2006-01-02"))
		printCSV(toDate, hdr)
	}
	if generateImage {
		printImage(toDate, hdr)
	}
}

func printImage(date time.Time, hdr *hdrhistogram.Histogram) error {
	p, err := plot.New()
	if err != nil {
		return err
	}

	p.Title.Text = fmt.Sprintf("%s's response time", system)
	p.Y.Label.Text = "response time"
	p.X.Label.Text = "percentile"
	p.Y.Tick.Marker = TimeTicks{}

	var rsp = make(plotter.XYs, len(percentiles))
	for i, p := range percentiles {
		rsp[i].X = p
		rsp[i].Y = float64(hdr.ValueAtQuantile(p)) / float64(1000)
	}

	err = plotutil.AddLinePoints(p, "response time", rsp)
	if err != nil {
		return err
	}

	outputPath := fmt.Sprintf("/tmp/%s/%s.png", system, date.Format("2006-01-02"))
	os.Mkdir(fmt.Sprintf("/tmp/%s", system), 0755)
	if err := p.Save(12*vg.Inch, 4*vg.Inch, outputPath); err != nil {
		return err
	}

	return nil
}

func printCSV(date time.Time, hdr *hdrhistogram.Histogram) error {
	for _, p := range percentiles {
		fmt.Printf(";%q;%q\n", fmt.Sprintf("%2.2f%%", p), fmt.Sprintf("%4.3f", float64(hdr.ValueAtQuantile(p))/float64(1000)))
	}
	return nil
}

type annotation struct {
	Value     string `json:"value"`
	Timestamp uint64 `json:"timestamp"`
}

type binaryAnnotation struct {
	Key      string `json:"key"`
	Value    string `json:"value"`
	Endpoint struct {
		ServiceName string `json:"serviceName"`
	} `json:"endpoint"`
}

type span struct {
	TraceID           string             `json:"traceId"`
	ID                string             `json:"id"`
	ParentID          string             `json:"parentId"`
	Duration          uint64             `json:"duration"`
	Annotations       []annotation       `json:"annotations"`
	BinaryAnnotations []binaryAnnotation `json:"binaryAnnotations"`
}

func minMaxDuration(date time.Time) (int64, int64, int64, error) {
	index := fmt.Sprintf("production-%s", date.Format("2006-01-02"))
	exists, err := client.IndexExists(index).Do()
	if !exists || err != nil {
		return 0, 0, 0, err
	}

	agg := elastic.NewMaxAggregation()
	agg.Field("duration")
	sr, err := client.Search().Index(index).Query(q).Aggregation("max_duration", agg).Do()
	if err != nil {
		return 0, 0, 0, err
	}
	res, found := sr.Aggregations.Max("max_duration")
	if !found {
		return 0, 0, 0, fmt.Errorf("Missing aggregation %q from result set", "max_duration")
	}

	if res.Value == nil {
		return 0, 0, 0, fmt.Errorf("Missing result value")
	}
	return 0, int64(*res.Value), sr.TotalHits(), nil
}

func histogram(date time.Time) (*hdrhistogram.Histogram, error) {
	minDuration, maxDuration, totalHits, err := minMaxDuration(date)
	if err != nil {
		return nil, err
	}

	index := fmt.Sprintf("production-%s", date.Format("2006-01-02"))
	hdr := hdrhistogram.New(int64(minDuration), int64(maxDuration), 1)
	start, end := int64(0), totalHits
	for start < end {
		results, err := client.Search().Index(index).Query(q).From(int(start)).Size(2000).Do()
		if err != nil {
			return nil, err
		}
		var sp span
		for _, item := range results.Each(reflect.TypeOf(sp)) {
			hdr.RecordValue(int64(item.(span).Duration))
		}
		start = start + results.TotalHits()

		if start >= end {
			break
		}
	}

	return hdr, nil
}

plot.go

package main

import (
	"fmt"
	"math"

	"github.com/gonum/plot"
)

type TimeTicks struct{}

var _ plot.Ticker = TimeTicks{}

func precisionOf(x float64) int {
	return int(math.Max(math.Ceil(-math.Log10(math.Abs(x))), displayPrecision))
}

func formatFloatTick(v float64, prec int) string {
	if v < 1000 {
		return fmt.Sprintf("%4.02f ms", v)
	}
	return fmt.Sprintf("%4.02f s", v/1000.0)
}

const displayPrecision = 4

func (TimeTicks) Ticks(min, max float64) (ticks []plot.Tick) {
	const SuggestedTicks = 3
	if max < min {
		panic("illegal range")
	}
	tens := math.Pow10(int(math.Floor(math.Log10(max - min))))
	n := (max - min) / tens
	for n < SuggestedTicks {
		tens /= 10
		n = (max - min) / tens
	}

	majorMult := int(n / SuggestedTicks)
	switch majorMult {
	case 7:
		majorMult = 6
	case 9:
		majorMult = 8
	}
	majorDelta := float64(majorMult) * tens
	val := math.Floor(min/majorDelta) * majorDelta
	prec := precisionOf(majorDelta)
	for val <= max {
		if val >= min && val <= max {
			ticks = append(ticks, plot.Tick{Value: val, Label: formatFloatTick(val, prec)})
		}
		if math.Nextafter(val, val+majorDelta) == val {
			break
		}
		val += majorDelta
	}

	minorDelta := majorDelta / 2
	switch majorMult {
	case 3, 6:
		minorDelta = majorDelta / 3
	case 5:
		minorDelta = majorDelta / 5
	}

	val = math.Floor(min/minorDelta) * minorDelta
	for val <= max {
		found := false
		for _, t := range ticks {
			if t.Value == val {
				found = true
			}
		}
		if val >= min && val <= max && !found {
			ticks = append(ticks, plot.Tick{Value: val})
		}
		if math.Nextafter(val, val+minorDelta) == val {
			break
		}
		val += minorDelta
	}
	return
}