P99 across all metrics ~= avg of P99 reported by every host

main.go

package main

import (
	"fmt"
	"sort"
	"math/rand"
)

func p(in []int, x float64) int {
	ints := make([]int, len(in))
	copy(ints, in)
	sort.Ints(ints)
	return ints[int(float64(len(ints))*x)]
	
}

func main() {
	hostCount := 1000
	valuesPerHost := 1000
	maxValue := 1000
	
	allValues := []int{}
	valuesByHost := map[int][]int{}
	
	rnd := rand.New(rand.NewSource(1000))
	zipf := rand.NewZipf(rnd, 1.1, 1, uint64(maxValue))
	
	for i := 0; i < hostCount; i++ {
		values := []int{}
		for j := 0; j < valuesPerHost; j++ {
			v := int(zipf.Uint64())
			values = append(values, v)
			allValues = append(allValues, v)
		}
		valuesByHost[i] = values
	}
	
	p99ForAll := p(allValues, 0.99)
	
	sumOfP99s := 0
	maxOfP99s := 0
	for _, v := range valuesByHost {
		p99 := p(v, 0.99)
		if p99 > maxOfP99s {
			maxOfP99s = p99
		}
		sumOfP99s += p99
	}

	
	avgOfP99s := sumOfP99s / len(valuesByHost)
	
	fmt.Printf("p99ForAll=%v, avgOfP99s=%v, maxOfP99s=%v\n", p99ForAll, avgOfP99s, maxOfP99s)
}