Skip to content

Instantly share code, notes, and snippets.

@serinuntius
Last active May 12, 2018
Embed
What would you like to do?
JSONのログをパースするベンチマーク
package graqt
import (
"testing"
)
import (
"bufio"
"bytes"
"encoding/json"
"os"
"github.com/json-iterator/go"
)
type Request struct {
Level string `json:"level"`
Ts float64 `json:"ts"`
Caller string `json:"caller"`
Msg string `json:"msg"`
Time float64 `json:"time"`
RequestID string `json:"request_id"`
Path string `json:"path"`
Method string `json:"method"`
}
//{"level":"info","ts":1525134349.9798086,"caller":"graqt/middleware.go:25","msg":"","time":0.013506292,"request_id":"d8523ba8-9948-4171-90ae-57f3b4649efa","path":"/user","method":"GET"}
type Requests []Request
// 1行読んでjsonをパースして、structに詰める
func BenchmarkReadlineToJson(b *testing.B) {
for i := 0; i < b.N; i++ {
var requests Requests
f, _ := os.Open("request.log")
scanner := bufio.NewScanner(f)
for scanner.Scan() {
var req Request
json.Unmarshal(scanner.Bytes(), &req)
requests = append(requests, req)
}
}
}
// 1行ずつ読み込んでjsonのarrayを作り、最後に1発だけjsonパースする
func BenchmarkCreateJsonBytesToJSONWithAppend(b *testing.B) {
for i := 0; i < b.N; i++ {
var requests Requests
f, _ := os.Open("request.log")
scanner := bufio.NewScanner(f)
first := true
var data []byte
for scanner.Scan() {
b := scanner.Bytes()
if first {
b = append(b, ',')
data = append([]byte(`[`), b...)
} else {
b = append(b, ',')
data = append(data, b...)
}
first = false
}
data[len(data)-1] = ']'
json.Unmarshal(data, &requests)
}
}
// ファイルサイズ分の[]byteを用意して、1行ずつ読み込んでjsonのarrayを作り、最後に1発だけjsonパースする
func BenchmarkCreateJsonBytesToJsonWithFileSize(b *testing.B) {
for i := 0; i < b.N; i++ {
var requests Requests
f, _ := os.Open("request.log")
stat, _ := f.Stat()
scanner := bufio.NewScanner(f)
first := true
// あらかじめfile sizeだけの[]byteを用意しておく
data := make([]byte, stat.Size())
for scanner.Scan() {
b := scanner.Bytes()
if first {
b = append(b, ',')
data = append([]byte(`[`), b...)
} else {
b = append(b, ',')
data = append(data, b...)
}
first = false
}
data[len(data)-1] = ']'
json.Unmarshal(data, &requests)
}
}
// append使うより、bytes.Buffer使った方が楽だよね
func BenchmarkCreateJsonBytesToJsonWithBytesBuffer(b *testing.B) {
for i := 0; i < b.N; i++ {
var requests Requests
f, _ := os.Open("request.log")
scanner := bufio.NewScanner(f)
var buf bytes.Buffer
buf.WriteByte('[')
for scanner.Scan() {
b := scanner.Bytes()
buf.Write(b)
buf.WriteByte(',')
}
data := buf.Bytes()
data[len(data)-1] = ']'
json.Unmarshal(data, &requests)
}
}
// jsoniter使ってみる
func BenchmarkCreateJsonBytesToJsonWithJsoniter(b *testing.B) {
for i := 0; i < b.N; i++ {
var requests Requests
var json = jsoniter.ConfigCompatibleWithStandardLibrary
f, _ := os.Open("request.log")
scanner := bufio.NewScanner(f)
var buf bytes.Buffer
buf.WriteByte('[')
for scanner.Scan() {
b := scanner.Bytes()
buf.Write(b)
buf.WriteByte(',')
}
data := buf.Bytes()
data[len(data)-1] = ']'
json.Unmarshal(data, &requests)
}
}
// makeで行数分のRequest作った方が速いはず
func BenchmarkCreateJsonBytesToJsonWithArraySize(b *testing.B) {
for i := 0; i < b.N; i++ {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
f, _ := os.Open("request.log")
scanner := bufio.NewScanner(f)
var buf bytes.Buffer
buf.WriteByte('[')
lineCount := 0
for scanner.Scan() {
b := scanner.Bytes()
buf.Write(b)
buf.WriteByte(',')
lineCount += 1
}
data := buf.Bytes()
data[len(data)-1] = ']'
requests := make([]Request, lineCount)
json.Unmarshal(data, &requests)
}
}
@serinuntius

This comment has been minimized.

Copy link
Owner Author

@serinuntius serinuntius commented May 12, 2018

BenchmarkCreateJsonBytesToJsonWithArraySize が一番速い

BenchmarkReadlineToJson-4                                    100          10902167 ns/op         2513180 B/op      23729 allocs/op
BenchmarkCreateJsonBytesToJSONWithAppend-4                   100          10217755 ns/op         3477959 B/op      16666 allocs/op
BenchmarkCreateJsonBytesToJsonWithFileSize-4                 100          10436011 ns/op         3920520 B/op      16668 allocs/op
BenchmarkCreateJsonBytesToJsonWithBytesBuffer-4              200          12417798 ns/op         2184089 B/op      16651 allocs/op
BenchmarkCreateJsonBytesToJsonWithJsoniter-4                 500           2868349 ns/op         2454464 B/op      17846 allocs/op
BenchmarkCreateJsonBytesToJsonWithArraySize-4                500           2661742 ns/op         1649510 B/op      17815 allocs/op
@serinuntius

This comment has been minimized.

Copy link
Owner Author

@serinuntius serinuntius commented May 12, 2018

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment