go test -bench .

xl-meta_test.go

/*
 * Minio Cloud Storage, (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"

	jsoniter "github.com/json-iterator/go"
	"github.com/valyala/fastjson"
)

/* Benchamarks to compare the performance of parsing of XLMeta for growing number of parts.
   Json.UnMarshal and Gjson ("github.com/tidwall/gjson") functions are the compared for the speed of UnMarshalling.

   For this purpose Sample xl.sjon data is created for  parts sizes 10,100,1000,2000,5000,10000 part and then
   its parsed using the current implemntation of json.Unmarshal and the new approach Gjson library which is under scrutiny
   in the current benchmarks
*/

type objectPartInfo struct {
	Number int    `json:"number"`
	Name   string `json:"name"`
	ETag   string `json:"etag"`
	Size   int64  `json:"size"`
}

type checkSumInfo struct {
	Name      string `json:"name"`
	Algorithm string `json:"algorithm"`
	Hash      string `json:"hash"`
}

// Constant indicates current bit-rot algo used when creating objects.
const (
	bitRotAlgo = "blake2b"
)

// statInfo - carries stat information of the object.
type statInfo struct {
	Size    int64     `json:"size"`    // Size of the object `xl.json`.
	ModTime time.Time `json:"modTime"` // ModTime of the object `xl.json`.
}

type erasureInfo struct {
	Algorithm    string         `json:"algorithm"`
	DataBlocks   int            `json:"data"`
	ParityBlocks int            `json:"parity"`
	BlockSize    int64          `json:"blockSize"`
	Index        int            `json:"index"`
	Distribution []int          `json:"distribution"`
	Checksum     []checkSumInfo `json:"checksum,omitempty"`
}

// A xlMetaV1 represents `xl.json` metadata header.
type xlMetaV1 struct {
	Version string   `json:"version"` // Version of the current `xl.json`.
	Format  string   `json:"format"`  // Format of the current `xl.json`.
	Stat    statInfo `json:"stat"`    // Stat of the current object `xl.json`.
	// Erasure coded info for the current object `xl.json`.
	Erasure erasureInfo `json:"erasure"`
	// Minio release tag for current object `xl.json`.
	Minio struct {
		Release string `json:"release"`
	} `json:"minio"`
	// Metadata map for current object `xl.json`.
	Meta map[string]string `json:"meta,omitempty"`
	// Captures all the individual object `xl.json`.
	Parts []objectPartInfo `json:"parts,omitempty"`
}

// newXLMetaV1 - initializes new xlMetaV1, adds version, allocates a fresh erasure info.
func newXLMetaV1() xlMetaV1 {
	xlMeta := xlMetaV1{}
	xlMeta.Version = "1.0.0"
	xlMeta.Format = "xl"
	xlMeta.Minio.Release = "1.0.0"
	xlMeta.Erasure = erasureInfo{
		Algorithm:    "klauspost/reedsolomon/vandermonde",
		DataBlocks:   5,
		ParityBlocks: 5,
		BlockSize:    10485760,
		Distribution: []int{9, 10, 1, 2, 3, 4, 5, 6, 7, 8},
	}
	xlMeta.Stat = statInfo{
		Size:    int64(20),
		ModTime: time.Now().UTC(),
	}
	// Set meta data.
	xlMeta.Meta = make(map[string]string)
	xlMeta.Meta["testKey1"] = "val1"
	xlMeta.Meta["testKey2"] = "val2"
	return xlMeta
}

func (m *xlMetaV1) AddObjectCheckSum(checkSumNum int, name string, hash string, algo string) {
	checkSum := checkSumInfo{
		Name:      name,
		Algorithm: algo,
		Hash:      hash,
	}
	m.Erasure.Checksum[checkSumNum] = checkSum
}

// AddObjectPart - add a new object part in order.
func (m *xlMetaV1) AddObjectPart(partNumber int, partName string, partETag string, partSize int64) {
	partInfo := objectPartInfo{
		Number: partNumber,
		Name:   partName,
		ETag:   partETag,
		Size:   partSize,
	}

	// Proceed to include new part info.
	m.Parts[partNumber] = partInfo
}

func getXLMetaBytes(totalParts int) []byte {
	xlSampleMeta := getSampleXLMeta(totalParts)
	xlMetaBytes, err := json.Marshal(xlSampleMeta)
	if err != nil {
		panic(err)
	}
	return xlMetaBytes
}

func getSampleXLMeta(totalParts int) xlMetaV1 {
	xlMeta := newXLMetaV1()
	// Number of checksum info == total parts.
	xlMeta.Erasure.Checksum = make([]checkSumInfo, totalParts)
	// total number of parts.
	xlMeta.Parts = make([]objectPartInfo, totalParts)
	for i := 0; i < totalParts; i++ {
		partName := "part." + strconv.Itoa(i+1)
		// hard coding hash and algo value for the checksum, Since we are benchmarking the parsing of xl.json the magnitude doesn't affect the test,
		// The magnitude doesn't make a difference, only the size does.
		xlMeta.AddObjectCheckSum(i, partName, "a23f5eff248c4372badd9f3b2455a285cd4ca86c3d9a570b091d3fc5cd7ca6d9484bbea3f8c5d8d4f84daae96874419eda578fd736455334afbac2c924b3915a", "blake2b")
		xlMeta.AddObjectPart(i, partName, "d3fdd79cc3efd5fe5c068d7be397934b", 67108864)
	}
	return xlMeta
}

func parseXLStat(v *fastjson.Value) (si statInfo, err error) {
	// obtain stat info.
	stat := statInfo{}
	st := v.GetObject("stat")
	modTimeB := st.Get("modTime").GetStringBytes()
	if modTimeB == nil {
		return si, errors.New("corrupted")
	}
	// fetching modTime.
	stat.ModTime, err = time.Parse(time.RFC3339, string(modTimeB))
	if err != nil {
		return si, err
	}

	// obtain Stat.Size .
	stat.Size = st.Get("size").GetInt64()
	return stat, nil
}

func parseXLVersion(v *fastjson.Value) string {
	return string(v.GetStringBytes("version"))
}

func parseXLFormat(v *fastjson.Value) string {
	return string(v.GetStringBytes("format"))
}

func parseXLRelease(v *fastjson.Value) string {
	return string(v.GetStringBytes("minio", "release"))
}

func parseXLErasureInfo(v *fastjson.Value) (erasureInfo, error) {
	erasure := erasureInfo{}
	// parse the xlV1Meta.Erasure.Distribution.
	es := v.GetObject("erasure")
	disResult := es.Get("distribution").GetArray()

	var err error
	distribution := make([]int, len(disResult))
	for i, dis := range disResult {
		distribution[i], err = dis.Int()
		if err != nil {
			return erasure, err
		}
	}
	erasure.Distribution = distribution

	erasure.Algorithm = string(es.Get("algorithm").GetStringBytes())
	erasure.DataBlocks = es.Get("data").GetInt()
	erasure.ParityBlocks = es.Get("parity").GetInt()
	erasure.BlockSize = es.Get("blockSize").GetInt64()
	erasure.Index = es.Get("index").GetInt()

	checkSumsResult := es.Get("checksum").GetArray()

	// Parse xlMetaV1.Erasure.Checksum array.
	checkSums := make([]checkSumInfo, len(checkSumsResult))
	for i, ck := range checkSumsResult {
		algBytes := ck.GetStringBytes("algorithm")
		if algBytes == nil {
			return erasure, errors.New("bitrot error")
		}
		nameBytes := ck.GetStringBytes("name")
		if nameBytes == nil {
			return erasure, errors.New("corrupted")
		}
		checkSums[i] = checkSumInfo{
			Name:      string(nameBytes),
			Algorithm: string(algBytes),
			Hash:      string(ck.GetStringBytes("hash")),
		}
	}
	erasure.Checksum = checkSums
	return erasure, nil
}

func parseXLParts(partsResult []*fastjson.Value) []objectPartInfo {
	partInfo := make([]objectPartInfo, len(partsResult))
	for i, p := range partsResult {
		partInfo[i] = objectPartInfo{
			Number: p.GetInt("number"),
			Name:   string(p.GetStringBytes("name")),
			ETag:   string(p.GetStringBytes("etag")),
			Size:   p.GetInt64("size"),
		}
	}
	return partInfo
}

func parseXLMetaMap(v *fastjson.Value) map[string]string {
	metaMap := make(map[string]string)
	// Get xlMetaV1.Meta map.
	v.GetObject("meta").Visit(func(k []byte, kv *fastjson.Value) {
		metaMap[string(k)] = string(kv.GetStringBytes())
	})
	return metaMap
}

// xl.json Parser pool
var xlParserPool fastjson.ParserPool

// Constructs XLMetaV1 using `fastjson` lib to retrieve each field.
func fastJSONMetaV1UnmarshalJSON(xlMetaBuf []byte) (xlMeta xlMetaV1, err error) {
	parser := xlParserPool.Get()
	defer xlParserPool.Put(parser)

	var v *fastjson.Value
	v, err = parser.ParseBytes(xlMetaBuf)
	if err != nil {
		return xlMeta, err
	}

	// obtain version.
	xlMeta.Version = parseXLVersion(v)
	// obtain format.
	xlMeta.Format = parseXLFormat(v)

	// Parse xlMetaV1.Stat .
	stat, err := parseXLStat(v)
	if err != nil {
		return xlMeta, err
	}

	xlMeta.Stat = stat
	// parse the xlV1Meta.Erasure fields.
	xlMeta.Erasure, err = parseXLErasureInfo(v)
	if err != nil {
		return xlMeta, err
	}

	// Check for scenario where checksum information missing for some parts.
	partsResult := v.GetArray("parts")
	if len(xlMeta.Erasure.Checksum) != len(partsResult) {
		return xlMeta, errors.New("corrupted")
	}

	// Parse the XL Parts.
	xlMeta.Parts = parseXLParts(partsResult)
	// Get the xlMetaV1.Realse field.
	xlMeta.Minio.Release = parseXLRelease(v)
	// parse xlMetaV1.
	xlMeta.Meta = parseXLMetaMap(v)

	return xlMeta, nil
}

func benchmarkParseUnmarshalN(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaBuf := getXLMetaBytes(totalParts)
	b.SetBytes(int64(len(xlMetaBuf)))
	b.ReportAllocs()
	parser := os.Getenv("PARSER")
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			switch parser {
			case "fastjson":
				fastJSONMetaV1UnmarshalJSON(xlMetaBuf)
			case "jsoniter":
				var unMarshalXLMeta xlMetaV1
				jsoniter.Unmarshal(xlMetaBuf, &unMarshalXLMeta)
			}
		}
	})
}

func BenchmarkParseUnmarshal(b *testing.B) {
	for _, n := range []int{
		10,
		100,
		500,
		1000,
		5000,
		10000,
	} {
		b.Run(fmt.Sprintf("N%d", n), func(b *testing.B) {
			benchmarkParseUnmarshalN(b, n)
		})
	}
}