Throughput test in Go's readline

results.txt

$ go test --bench .
goos: linux
goarch: amd64
BenchmarkLines/10000000B/25/ragged:false         	     100	  11124454 ns/op	 898.92 MB/s
BenchmarkLines/10000000B/100/ragged:false        	     300	   4151054 ns/op	2409.03 MB/s
BenchmarkLines/10000000B/1000/ragged:false       	    1000	   1321200 ns/op	7568.87 MB/s
BenchmarkLines/10000000B/10000/ragged:false      	    2000	   1162743 ns/op	8600.35 MB/s
BenchmarkLines/100000000B/25/ragged:false        	      10	 113038763 ns/op	 884.65 MB/s
BenchmarkLines/100000000B/100/ragged:false       	      30	  40415252 ns/op	2474.31 MB/s
BenchmarkLines/100000000B/1000/ragged:false      	     100	  18977939 ns/op	5269.28 MB/s
BenchmarkLines/100000000B/10000/ragged:false     	     100	  18680743 ns/op	5353.11 MB/s
PASS
ok  	_/home/ondrej/tmp/perf	74.292s

throughput_test.go

package throughput

import (
	"bufio"
	"bytes"
	"fmt"
	"math/rand"
	"testing"
)

func genData(size, lineLength int, ragged bool) []byte {
	rand.Seed(50)
	data := make([]byte, 0, size)
	for j := 0; j < size; j++ {
		rnd := rand.Intn(26)
		data = append(data, 'a'+uint8(rnd))
		if j%lineLength == 0 && (!ragged || (ragged && rand.Float32() > .5)) {
			data = append(data, '\n')
		}
	}
	return data
}

func BenchmarkLines(b *testing.B) {
	for _, props := range []struct {
		size, lineLength int
		ragged           bool
	}{
		// cache friendly
		{10 * 1000 * 1000, 25, false}, // similar to Lemire's length of line
		{10 * 1000 * 1000, 100, false},
		{10 * 1000 * 1000, 1000, false},
		{10 * 1000 * 1000, 10000, false},

		// larger than even L3
		{100 * 1000 * 1000, 25, false},
		{100 * 1000 * 1000, 100, false},
		{100 * 1000 * 1000, 1000, false},
		{100 * 1000 * 1000, 10000, false},
	} {
		name := fmt.Sprintf("%vB/%v/ragged:%v", props.size, props.lineLength, props.ragged)
		b.Run(name, func(b *testing.B) {
			data := genData(props.size, props.lineLength, props.ragged)
			b.ResetTimer()
			for j := 0; j < b.N; j++ {
				rd := bytes.NewReader(data)
				sc := bufio.NewScanner(rd)

				var n int64
				for sc.Scan() {
					n += int64(len(sc.Bytes()))
				}
				if sc.Err() != nil {
					b.Error(sc.Err())
				}
				b.SetBytes(n)
			}
		})
	}

}