JSONのログをパースするベンチマーク

json_test.go

package graqt

import (
	"testing"
)

import (
	"bufio"
	"bytes"
	"encoding/json"
	"os"

	"github.com/json-iterator/go"
)

type Request struct {
	Level     string  `json:"level"`
	Ts        float64 `json:"ts"`
	Caller    string  `json:"caller"`
	Msg       string  `json:"msg"`
	Time      float64 `json:"time"`
	RequestID string  `json:"request_id"`
	Path      string  `json:"path"`
	Method    string  `json:"method"`
}

//{"level":"info","ts":1525134349.9798086,"caller":"graqt/middleware.go:25","msg":"","time":0.013506292,"request_id":"d8523ba8-9948-4171-90ae-57f3b4649efa","path":"/user","method":"GET"}

type Requests []Request

// １行読んでjsonをパースして、structに詰める
func BenchmarkReadlineToJson(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var requests Requests
		f, _ := os.Open("request.log")
		scanner := bufio.NewScanner(f)

		for scanner.Scan() {
			var req Request

			json.Unmarshal(scanner.Bytes(), &req)
			requests = append(requests, req)
		}
	}
}

// １行ずつ読み込んでjsonのarrayを作り、最後に１発だけjsonパースする

func BenchmarkCreateJsonBytesToJSONWithAppend(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var requests Requests
		f, _ := os.Open("request.log")

		scanner := bufio.NewScanner(f)
		first := true
		var data []byte

		for scanner.Scan() {
			b := scanner.Bytes()

			if first {
				b = append(b, ',')
				data = append([]byte(`[`), b...)
			} else {
				b = append(b, ',')
				data = append(data, b...)
			}
			first = false
		}
		data[len(data)-1] = ']'

		json.Unmarshal(data, &requests)

	}
}

// ファイルサイズ分の[]byteを用意して、１行ずつ読み込んでjsonのarrayを作り、最後に１発だけjsonパースする
func BenchmarkCreateJsonBytesToJsonWithFileSize(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var requests Requests

		f, _ := os.Open("request.log")
		stat, _ := f.Stat()

		scanner := bufio.NewScanner(f)
		first := true

		// あらかじめfile sizeだけの[]byteを用意しておく
		data := make([]byte, stat.Size())

		for scanner.Scan() {
			b := scanner.Bytes()

			if first {
				b = append(b, ',')
				data = append([]byte(`[`), b...)
			} else {
				b = append(b, ',')
				data = append(data, b...)
			}
			first = false
		}
		data[len(data)-1] = ']'

		json.Unmarshal(data, &requests)
	}
}

// append使うより、bytes.Buffer使った方が楽だよね
func BenchmarkCreateJsonBytesToJsonWithBytesBuffer(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var requests Requests
		f, _ := os.Open("request.log")

		scanner := bufio.NewScanner(f)
		var buf bytes.Buffer
		buf.WriteByte('[')

		for scanner.Scan() {
			b := scanner.Bytes()
			buf.Write(b)
			buf.WriteByte(',')

		}

		data := buf.Bytes()
		data[len(data)-1] = ']'

		json.Unmarshal(data, &requests)
	}
}

// jsoniter使ってみる
func BenchmarkCreateJsonBytesToJsonWithJsoniter(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var requests Requests
		var json = jsoniter.ConfigCompatibleWithStandardLibrary
		f, _ := os.Open("request.log")

		scanner := bufio.NewScanner(f)

		var buf bytes.Buffer
		buf.WriteByte('[')

		for scanner.Scan() {
			b := scanner.Bytes()
			buf.Write(b)
			buf.WriteByte(',')
		}

		data := buf.Bytes()
		data[len(data)-1] = ']'

		json.Unmarshal(data, &requests)
	}
}

// makeで行数分のRequest作った方が速いはず
func BenchmarkCreateJsonBytesToJsonWithArraySize(b *testing.B) {
	for i := 0; i < b.N; i++ {
		var json = jsoniter.ConfigCompatibleWithStandardLibrary
		f, _ := os.Open("request.log")

		scanner := bufio.NewScanner(f)

		var buf bytes.Buffer
		buf.WriteByte('[')
		lineCount := 0

		for scanner.Scan() {
			b := scanner.Bytes()

			buf.Write(b)
			buf.WriteByte(',')

			lineCount += 1
		}

		data := buf.Bytes()
		data[len(data)-1] = ']'

		requests := make([]Request, lineCount)

		json.Unmarshal(data, &requests)
	}
}

BenchmarkCreateJsonBytesToJsonWithArraySize が一番速い

BenchmarkReadlineToJson-4                                    100          10902167 ns/op         2513180 B/op      23729 allocs/op
BenchmarkCreateJsonBytesToJSONWithAppend-4                   100          10217755 ns/op         3477959 B/op      16666 allocs/op
BenchmarkCreateJsonBytesToJsonWithFileSize-4                 100          10436011 ns/op         3920520 B/op      16668 allocs/op
BenchmarkCreateJsonBytesToJsonWithBytesBuffer-4              200          12417798 ns/op         2184089 B/op      16651 allocs/op
BenchmarkCreateJsonBytesToJsonWithJsoniter-4                 500           2868349 ns/op         2454464 B/op      17846 allocs/op
BenchmarkCreateJsonBytesToJsonWithArraySize-4                500           2661742 ns/op         1649510 B/op      17815 allocs/op

request.logはこちらにあります。
https://gist.github.com/serinuntius/8ab001ca0197147a73c606af100804e2