Dieterbe

#!/bin/bash

read -r -d '' text <<'EOF'
'" <> ?+| /=\ -_ ,.:;
ABCDEFGHIJKLMNOPQRSTUVWXYZ
abcdefghijklmnopqrstuvwxyz
! @ # $ % ^ & * ( ) { }
1 2 3 4 5 6 7 8 9 0 [ ]
EOF
echo -e "Regular:\n$text"

Dieterbe

#!/bin/bash
# useful articles:
# https://vstinner.github.io/journey-to-stable-benchmark-system.html 
# http://perf.readthedocs.io/en/latest/system.html
# https://baiweiblog.wordpress.com/2017/11/02/how-to-set-processor-affinity-in-linux-using-taskset/
# https://wiki.archlinux.org/index.php/CPU_frequency_scaling#CPU_frequency_driver

if [[ $EUID > 0 ]]; then
  echo "Please run as root"
  exit 1

Dieterbe

# start nsqd:
nsqd -max-msg-size=10024768 -sync-every=1 --mem-queue-size=0 -e2e-processing-latency-percentile=0.90,0.99 -e2e-processing-latency-window-time=10s -statsd-address=localhost:8125 -statsd-interval=10s

# start consumer:
nsq_tail -nsqd-tcp-address=127.0.0.1:4150 -topic=test -channel=nsq_tail > /dev/null

# start producer when ready:
while true; do head -c 600000 /dev/zero|tr '\0' 'A'; echo; done | to_nsq -nsqd-tcp-address=127.0.0.1:4150 --topic=test

## results:

Dieterbe

{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "limit": 100,

Dieterbe

#!/bin/bash
# useful articles:
# https://vstinner.github.io/journey-to-stable-benchmark-system.html 
# http://perf.readthedocs.io/en/latest/system.html
# https://baiweiblog.wordpress.com/2017/11/02/how-to-set-processor-affinity-in-linux-using-taskset/
# https://wiki.archlinux.org/index.php/CPU_frequency_scaling#CPU_frequency_driver

if [[ $EUID > 0 ]]; then
  echo "Please run as root"
  exit 1

Dieterbe

$ go test -count=10 -run='^$' -bench=. -benchtime=100ms ./... 
goos: linux
goarch: amd64
pkg: github.com/grafana/metrictank/api
BenchmarkFix1M-8           	      16	   9189930 ns/op	1305.78 MB/s
BenchmarkFix1M-8           	      14	   8155018 ns/op	1471.49 MB/s
BenchmarkFix1M-8           	      13	  12086950 ns/op	 992.81 MB/s
BenchmarkFix1M-8           	      13	   9010493 ns/op	1331.78 MB/s
BenchmarkFix1M-8           	      16	  10516333 ns/op	1141.08 MB/s
BenchmarkFix1M-8           	      16	   9929228 ns/op	1208.55 MB/s

Dieterbe

as described in devdocs/expr.md, metrictank resets the consolidation function when it passes a "special" function. a special function is one that changes the nature of the data on a fundamental level. Such as summarize, perSecond, derivative, integral We explain the reasoning for that behavior with a few examples.

example 1

summarize(consolidateBy(statsd.count,"sum"),"24h","sum") 

• the inputs to summarize, if we need to consolidate, can be summed, because we'll sum the data anyway
• but if we want to see daily sums, and maxdatapoints dictates that we must consolidate at runtime we don't want the points summed together into buckets of arbitrary N-day-size buckets.

Dieterbe

              sarama/8           rtm      3424 -> 0.030103
              sarama/8            id    687540 -> 0.002537
              sarama/8           ops   1009213 -> 0.004118
              sarama/8           fng   3024891 -> 0.001198
              sarama/8    fake-false      1000 -> 0.032249
              sarama/8    fake-false     10000 -> 0.006223
              sarama/8    fake-false    100000 -> 0.002262
              sarama/8    fake-false   1000000 -> 0.000801
              sarama/8    fake-false  10000000 -> 0.000325
              sarama/8     fake-true      1000 -> 1.088978

Dieterbe

taskset --cpu-list 1,2 go test . -run='^$' -test.benchmem -bench .
goos: linux
goarch: amd64
pkg: github.com/grafana/metrictank/cluster/partitioner
BenchmarkPartitionerSarama-2       	10000000	       205 ns/op	       0 B/op	       0 allocs/op
BenchmarkJumpPartitionerMauro-2    	 5000000	       271 ns/op	     210 B/op	       1 allocs/op
BenchmarkJumpPartitionerFnv-2      	 5000000	       323 ns/op	       8 B/op	       1 allocs/op
BenchmarkJumpPartitionerMetro-2    	20000000	       101 ns/op	       0 B/op	       0 allocs/op
BenchmarkJumpPartitionerSip-2      	10000000	       145 ns/op	       0 B/op	       0 allocs/op
BenchmarkJumpPartitionerXxhash-2   	20000000	        99.4 ns/op	       0 B/op	       0 allocs/op

Dieterbe

unused ./...
Unused has been deprecated. Please use staticcheck instead.
api/config.go:20:2: var logMinDur is unused (U1000)
cmd/mt-store-cat/match_test.go:5:6: type matchCase is unused (U1000)
cmd/mt-store-cat/match_test.go:40:6: func testmatch is unused (U1000)
cmd/mt-update-ttl/main_test.go:5:6: type testCase is unused (U1000)
cmd/mt-update-ttl/main_test.go:28:6: type completenessCase is unused (U1000)
cmd/mt-whisper-importer-reader/conversion_test.go:10:6: func testIncResolution is unused (U1000)
cmd/mt-whisper-importer-reader/conversion_test.go:397:6: func testDecResolution is unused (U1000)
cmd/mt-whisper-importer-reader/conversion_test.go:422:6: func getSimpleInData is unused (U1000)