golang testing bufio.Scanner and strconv.Atoi versus fmt.Scan

gistfile1.txt

package main

// These benchmarks are used to compare how fast it is to read a set of space separated numbers from a string
// using different techniques.

// The primary motiviation for this came when writing some code for hackerrank and realizing that the parsing of numbers
// for large input sets was actually the bottleneck
//
// For stable results, it is recommended to increase the runtime allocated for benchmarking, such as
// go test -v -run none -bench . -benchtime 5s
//
//
// sample results on a 2011 iMac
// go test -v -run none -bench . -benchtime 5s
// BenchmarkScannerAtoi	30000000	       261 ns/op
// BenchmarkFscan	 5000000	      1444 ns/op

import (
	"bufio"
	"bytes"
	"fmt"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"testing"
)

// numberList consists of a text list of random numbers and a total of the numbers
// this allows the scanning routines to validate they read all the numbers correctly
type numberList struct {
	// text consists of space separated random ints of up to 31 bit
	text string
	// total of all the numbers in text field
	total int64
}

var numbers = make(map[int]numberList)
var mu sync.Mutex

// genNumbers will return a numberList of suitable size (from cache if previously requested)
// to make sure the generated lists are of equal complexity for all tests
// and to avoid re-computing lists between benchmark functions
func genNumbers(n int) numberList {
	mu.Lock()
	defer mu.Unlock()
	if val, exists := numbers[n]; exists {
		return val
	}
	var buf bytes.Buffer
	var total int64
	for i := 0; i < n-1; i++ {
		tmp := int(rand.Int31())
		total += int64(tmp)
		buf.WriteString(strconv.Itoa(tmp) + " ")
	}
	tmp := int(rand.Int31())
	buf.WriteString(strconv.Itoa(tmp))
	total += int64(tmp)
	nl := numberList{text: buf.String(), total: total}
	numbers[n] = nl
	return nl
}

// BenchmarkScannerAtoi will read a string of b.N numbers using a bufio.Scanner and strconv.Atoi
// we leave the NewScanner/Split overhead in place since this is not needed by Fscan
func BenchmarkScannerAtoi(b *testing.B) {
	nl := genNumbers(b.N)
	reader := strings.NewReader(nl.text)
	b.ResetTimer()
	s := bufio.NewScanner(reader)
	s.Split(bufio.ScanWords)
	var total int64
	var val int
	var err error
	for i := 0; i < b.N; i++ {
		if !s.Scan() {
			b.Fatal("s.Scan returned false")
		}
		val, err = strconv.Atoi(s.Text())
		if err != nil {
			b.Fatal(err)
		}
		total += int64(val)
	}
	if total != nl.total {
		b.Fatalf("total didn't match found=%d expected=%d\n", total, nl.total)
	}
}

// BenchmarkFscan will read a string of b.N numbers using fmt.Fscan
func BenchmarkFscan(b *testing.B) {
	nl := genNumbers(b.N)
	reader := strings.NewReader(nl.text)
	b.ResetTimer()
	var total int64
	var val int
	var err error
	for i := 0; i < b.N; i++ {
		_, err = fmt.Fscan(reader, &val)
		if err != nil {
			b.Fatal(err)
		}
		total += int64(val)
	}
	if total != nl.total {
		b.Fatalf("total didn't match found=%d expected=%d\n", total, nl.total)
	}
}