Skip to content

Instantly share code, notes, and snippets.

@nicolai86
Last active December 25, 2016 17:14
Show Gist options
  • Save nicolai86/41dc3d9beaabf7668d09298487dafebe to your computer and use it in GitHub Desktop.
Save nicolai86/41dc3d9beaabf7668d09298487dafebe to your computer and use it in GitHub Desktop.
zipkin ES response percentiles
package main
import (
"flag"
"fmt"
"os"
"sort"
"time"
"github.com/gonum/plot"
"github.com/gonum/plot/plotter"
"github.com/gonum/plot/plotutil"
"github.com/gonum/plot/vg"
elastic "gopkg.in/olivere/elastic.v3"
)
var (
system string
client *elastic.Client
q elastic.Query
indexPrefix string
generateCSV bool
generateImage bool
percentiles = []float64{
10.0,
20.0,
30.0,
40.0,
50.0,
55.0,
60.0,
65.0,
70.0,
75.0,
77.5,
80.0,
82.5,
85.0,
87.5,
88.75,
90.0,
91.25,
92.5,
93.75,
94.375,
95.0,
95.625,
96.25,
96.875,
97.1875,
97.5,
97.8125,
98.125,
98.4375,
98.5938,
98.75,
98.9062,
99.0625,
99.2188,
99.2969,
99.3750,
99.4531,
99.5313,
99.6094,
99.6484,
99.6875,
99.7266,
99.7656,
99.8047,
99.8242,
99.8437,
99.8633,
99.8828,
99.9023,
99.9121,
99.9219,
99.9316,
99.9414,
99.9512,
99.9561,
99.9609,
99.9658,
99.9707,
99.9756,
99.9780,
99.9805,
99.9829,
99.9854,
99.9878,
99.9890,
99.9902,
}
)
func init() {
flag.StringVar(&indexPrefix, "prefix", "", "elasticsearch index prefix")
flag.BoolVar(&generateCSV, "csv", true, "generate csv data")
flag.BoolVar(&generateImage, "img", false, "generate image data")
flag.Parse()
system = flag.Args()[0]
b := elastic.NewBoolQuery()
b.Must(elastic.NewMatchQuery("binaryAnnotations.endpoint.serviceName", system))
q = elastic.NewNestedQuery(
"binaryAnnotations",
b,
)
c, err := elastic.NewClient(
elastic.SetURL("127.0.0.1:9200"),
elastic.SetHealthcheck(false),
elastic.SetSniff(false),
)
if c == nil || err != nil {
panic(fmt.Errorf("Failed retrieving an client: %#v", err))
}
client = c
}
func main() {
fromDate, err := time.Parse("2006-01-02", flag.Args()[1])
if err != nil {
panic(err)
}
toDate, err := time.Parse("2006-01-02", flag.Args()[2])
if err != nil {
panic(err)
}
date := fromDate
if generateCSV {
fmt.Printf("%q;%q;%q\n", "date", "percentile", "response time in ms")
}
for date.Before(toDate) {
hdr, err := durationPercentiles(date)
if err == nil {
if generateCSV {
fmt.Printf("%q;;\n", date.Format("2006-01-02"))
printCSV(date, hdr)
}
if generateImage {
printImage(date, hdr)
}
} else {
panic(err)
}
date = date.Add(time.Duration(24) * time.Hour)
}
hdr, err := durationPercentiles(toDate)
if err != nil {
panic(err)
}
if generateCSV {
fmt.Printf("%q;;\n", toDate.Format("2006-01-02"))
printCSV(toDate, hdr)
}
if generateImage {
printImage(toDate, hdr)
}
}
func printImage(date time.Time, hdr []float64) error {
p, err := plot.New()
if err != nil {
return err
}
p.Title.Text = fmt.Sprintf("%s's response time", system)
p.Y.Label.Text = "response time"
p.X.Label.Text = "percentile"
p.Y.Tick.Marker = TimeTicks{}
var rsp = make(plotter.XYs, len(percentiles))
for i, p := range percentiles {
rsp[i].X = p
rsp[i].Y = hdr[i] / float64(1000)
}
err = plotutil.AddLinePoints(p, "response time", rsp)
if err != nil {
return err
}
outputPath := fmt.Sprintf("/tmp/%s/%s.png", system, date.Format("2006-01-02"))
os.Mkdir(fmt.Sprintf("/tmp/%s", system), 0755)
if err := p.Save(12*vg.Inch, 4*vg.Inch, outputPath); err != nil {
return err
}
return nil
}
func printCSV(date time.Time, hdr []float64) error {
for i, p := range percentiles {
fmt.Printf(";%q;%q\n", fmt.Sprintf("%2.2f%%", p), fmt.Sprintf("%4.3f", hdr[i]/float64(1000)))
}
return nil
}
type annotation struct {
Value string `json:"value"`
Timestamp uint64 `json:"timestamp"`
}
type binaryAnnotation struct {
Key string `json:"key"`
Value string `json:"value"`
Endpoint struct {
ServiceName string `json:"serviceName"`
} `json:"endpoint"`
}
type span struct {
TraceID string `json:"traceId"`
ID string `json:"id"`
ParentID string `json:"parentId"`
Duration uint64 `json:"duration"`
Annotations []annotation `json:"annotations"`
BinaryAnnotations []binaryAnnotation `json:"binaryAnnotations"`
}
type hdrAgg struct {
*elastic.PercentilesAggregation
}
func (c *hdrAgg) Source() (interface{}, error) {
s, err := c.PercentilesAggregation.Source()
if err != nil {
return nil, err
}
m := s.(map[string]interface{})
m2 := m["percentiles"].(map[string]interface{})
m2["hdr"] = map[string]interface{}{
"number_of_significant_value_digits": 3,
}
return m, nil
}
func durationPercentiles(date time.Time) ([]float64, error) {
index := fmt.Sprintf("%s%s", indexPrefix, date.Format("2006-01-02"))
exists, err := client.IndexExists(index).Do()
if !exists || err != nil {
return nil, err
}
agg := elastic.NewPercentilesAggregation()
agg.Field("duration")
agg.Percentiles(percentiles...)
sr, err := client.Search().Index(index).Query(q).Aggregation("duration", &hdrAgg{agg}).Do()
if err != nil {
return nil, err
}
res, found := sr.Aggregations.Percentiles("duration")
if !found {
return nil, fmt.Errorf("Missing aggregation %q from result set", "duration")
}
if res.Values == nil {
return nil, fmt.Errorf("Missing result value")
}
var keys []string
for k := range res.Values {
keys = append(keys, k)
}
sort.Strings(keys)
values := []float64{}
for _, k := range keys {
values = append(values, res.Values[k])
}
return values, nil
}
package main
import (
"fmt"
"math"
"github.com/gonum/plot"
)
type TimeTicks struct{}
var _ plot.Ticker = TimeTicks{}
func precisionOf(x float64) int {
return int(math.Max(math.Ceil(-math.Log10(math.Abs(x))), displayPrecision))
}
func formatFloatTick(v float64, prec int) string {
if v < 1000 {
return fmt.Sprintf("%4.02f ms", v)
}
return fmt.Sprintf("%4.02f s", v/1000.0)
}
const displayPrecision = 4
func (TimeTicks) Ticks(min, max float64) (ticks []plot.Tick) {
const SuggestedTicks = 3
if max < min {
panic("illegal range")
}
tens := math.Pow10(int(math.Floor(math.Log10(max - min))))
n := (max - min) / tens
for n < SuggestedTicks {
tens /= 10
n = (max - min) / tens
}
majorMult := int(n / SuggestedTicks)
switch majorMult {
case 7:
majorMult = 6
case 9:
majorMult = 8
}
majorDelta := float64(majorMult) * tens
val := math.Floor(min/majorDelta) * majorDelta
prec := precisionOf(majorDelta)
for val <= max {
if val >= min && val <= max {
ticks = append(ticks, plot.Tick{Value: val, Label: formatFloatTick(val, prec)})
}
if math.Nextafter(val, val+majorDelta) == val {
break
}
val += majorDelta
}
minorDelta := majorDelta / 2
switch majorMult {
case 3, 6:
minorDelta = majorDelta / 3
case 5:
minorDelta = majorDelta / 5
}
val = math.Floor(min/minorDelta) * minorDelta
for val <= max {
found := false
for _, t := range ticks {
if t.Value == val {
found = true
}
}
if val >= min && val <= max && !found {
ticks = append(ticks, plot.Tick{Value: val})
}
if math.Nextafter(val, val+minorDelta) == val {
break
}
val += minorDelta
}
return
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment