Skip to content

Instantly share code, notes, and snippets.

@kokes
Last active July 27, 2019 13:42
Show Gist options
  • Save kokes/29b9e81f55b61e8950e751c69b709ebf to your computer and use it in GitHub Desktop.
Save kokes/29b9e81f55b61e8950e751c69b709ebf to your computer and use it in GitHub Desktop.
Throughput test in Go's readline
$ go test --bench .
goos: linux
goarch: amd64
BenchmarkLines/10000000B/25/ragged:false 100 11124454 ns/op 898.92 MB/s
BenchmarkLines/10000000B/100/ragged:false 300 4151054 ns/op 2409.03 MB/s
BenchmarkLines/10000000B/1000/ragged:false 1000 1321200 ns/op 7568.87 MB/s
BenchmarkLines/10000000B/10000/ragged:false 2000 1162743 ns/op 8600.35 MB/s
BenchmarkLines/100000000B/25/ragged:false 10 113038763 ns/op 884.65 MB/s
BenchmarkLines/100000000B/100/ragged:false 30 40415252 ns/op 2474.31 MB/s
BenchmarkLines/100000000B/1000/ragged:false 100 18977939 ns/op 5269.28 MB/s
BenchmarkLines/100000000B/10000/ragged:false 100 18680743 ns/op 5353.11 MB/s
PASS
ok _/home/ondrej/tmp/perf 74.292s
package throughput
import (
"bufio"
"bytes"
"fmt"
"math/rand"
"testing"
)
func genData(size, lineLength int, ragged bool) []byte {
rand.Seed(50)
data := make([]byte, 0, size)
for j := 0; j < size; j++ {
rnd := rand.Intn(26)
data = append(data, 'a'+uint8(rnd))
if j%lineLength == 0 && (!ragged || (ragged && rand.Float32() > .5)) {
data = append(data, '\n')
}
}
return data
}
func BenchmarkLines(b *testing.B) {
for _, props := range []struct {
size, lineLength int
ragged bool
}{
// cache friendly
{10 * 1000 * 1000, 25, false}, // similar to Lemire's length of line
{10 * 1000 * 1000, 100, false},
{10 * 1000 * 1000, 1000, false},
{10 * 1000 * 1000, 10000, false},
// larger than even L3
{100 * 1000 * 1000, 25, false},
{100 * 1000 * 1000, 100, false},
{100 * 1000 * 1000, 1000, false},
{100 * 1000 * 1000, 10000, false},
} {
name := fmt.Sprintf("%vB/%v/ragged:%v", props.size, props.lineLength, props.ragged)
b.Run(name, func(b *testing.B) {
data := genData(props.size, props.lineLength, props.ragged)
b.ResetTimer()
for j := 0; j < b.N; j++ {
rd := bytes.NewReader(data)
sc := bufio.NewScanner(rd)
var n int64
for sc.Scan() {
n += int64(len(sc.Bytes()))
}
if sc.Err() != nil {
b.Error(sc.Err())
}
b.SetBytes(n)
}
})
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment