metric_test.go

metric_test.go

package vegeta

import (
	"math/rand"
	"reflect"
	"testing"
	"time"

	axiom "github.com/axiomhq/quantiles"
	bmizerany "github.com/bmizerany/perks/quantile"
	gk "github.com/dgryski/go-gk"
	streadway "github.com/streadway/quantile"
)

func TestMetrics_Add(t *testing.T) {
	t.Parallel()

	codes := []uint16{500, 200, 302}
	errors := []string{"Internal server error", ""}

	var got Metrics
	for i := 1; i <= 10000; i++ {
		got.Add(&Result{
			Code:      codes[i%len(codes)],
			Timestamp: time.Unix(int64(i-1), 0),
			Latency:   time.Duration(i) * time.Microsecond,
			BytesIn:   1024,
			BytesOut:  512,
			Error:     errors[i%len(errors)],
		})
	}
	got.Close()

	duration := func(s string) time.Duration {
		d, err := time.ParseDuration(s)
		if err != nil {
			panic(err)
		}
		return d
	}

	want := Metrics{
		Latencies: LatencyMetrics{
			Total:     duration("50.005s"),
			Mean:      duration("5.0005ms"),
			P50:       duration("5.0005ms"),
			P95:       duration("9.5005ms"),
			P99:       duration("9.9005ms"),
			Max:       duration("10ms"),
			estimator: got.Latencies.estimator,
		},
		BytesIn:     ByteMetrics{Total: 10240000, Mean: 1024},
		BytesOut:    ByteMetrics{Total: 5120000, Mean: 512},
		Earliest:    time.Unix(0, 0),
		Latest:      time.Unix(9999, 0),
		End:         time.Unix(9999, 0).Add(10000 * time.Microsecond),
		Duration:    duration("2h46m39s"),
		Wait:        duration("10ms"),
		Requests:    10000,
		Rate:        1.000100010001,
		Success:     0.6667,
		StatusCodes: map[string]int{"500": 3333, "200": 3334, "302": 3333},
		Errors:      []string{"Internal server error"},

		errors:  got.errors,
		success: got.success,
	}

	if !reflect.DeepEqual(got, want) {
		t.Errorf("\ngot:  %+v\nwant: %+v", got, want)
	}
}

// https://github.com/tsenart/vegeta/issues/208
func TestMetrics_NoInfiniteRate(t *testing.T) {
	t.Parallel()

	m := Metrics{Requests: 1, Duration: time.Microsecond}
	m.Close()

	if got, want := m.Rate, 1.0; got != want {
		t.Errorf("got rate %f, want %f", got, want)
	}
}

// https://github.com/tsenart/vegeta/pull/277
func TestMetrics_NonNilErrorsOnClose(t *testing.T) {
	t.Parallel()

	m := Metrics{Errors: nil}
	m.Close()

	got, want := m.Errors, []string{}

	if !reflect.DeepEqual(got, want) {
		t.Errorf("\ngot:  %+v\nwant: %+v", got, want)
	}
}

func BenchmarkMetrics(b *testing.B) {
	b.StopTimer()
	b.ResetTimer()

	rng := rand.New(rand.NewSource(time.Now().UnixNano()))

	latencies := make([]time.Duration, 1000000)
	for i := range latencies {
		latencies[i] = time.Duration(1e6 + rng.Int63n(1e10-1e6)) // 1ms to 10s
	}

	for _, tc := range []struct {
		name string
		estimator
	}{
		{"streadway/quantile", streadway.New(
			streadway.Known(0.50, 0.01),
			streadway.Known(0.95, 0.001),
			streadway.Known(0.99, 0.0005),
		)},
		{"bmizerany/perks/quantile", newBmizeranyEstimator(
			0.50,
			0.95,
			0.99,
		)},
		{"dgrisky/go-gk", newDgriskyEstimator(0.5)},
		{"influxdata/tdigest", newTdigestEstimator(100)},
		{"axiom/quantiles", newAxiomEstimator(0.5)},
	} {
		m := Metrics{Latencies: LatencyMetrics{estimator: tc.estimator}}
		b.Run("Add/"+tc.name, func(b *testing.B) {
			for i := 0; i <= b.N; i++ {
				m.Add(&Result{
					Code:      200,
					Timestamp: time.Unix(int64(i), 0),
					Latency:   latencies[i%len(latencies)],
					BytesIn:   1024,
					BytesOut:  512,
				})
			}

		})

		b.Run("Close/"+tc.name, func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				m.Close()
			}
		})
	}

}

type bmizeranyEstimator struct {
	*bmizerany.Stream
}

func newBmizeranyEstimator(qs ...float64) *bmizeranyEstimator {
	return &bmizeranyEstimator{Stream: bmizerany.NewTargeted(qs...)}
}

func (e *bmizeranyEstimator) Add(s float64) { e.Insert(s) }
func (e *bmizeranyEstimator) Get(q float64) float64 {
	return e.Query(q)
}

type dgryskiEstimator struct {
	*gk.Stream
}

func newDgriskyEstimator(epsilon float64) *dgryskiEstimator {
	return &dgryskiEstimator{Stream: gk.New(epsilon)}
}

func (e *dgryskiEstimator) Add(s float64) { e.Insert(s) }
func (e *dgryskiEstimator) Get(q float64) float64 {
	return e.Query(q)
}

type axiomEstimator struct {
	*axiom.Sketch
}

func newAxiomEstimator(epsilon float64) *axiomEstimator {
	sketch, _ := axiom.New(epsilon, 1000)
	return &axiomEstimator{Sketch: sketch}
}

func (e *axiomEstimator) Add(s float64) { e.Push(s, 1.0) }
func (e *axiomEstimator) Get(q float64) float64 {
	// ignore finalized error if its called once then all good
	e.Finalize()
	val, _ := e.Quantile(q)
	return val
}