comparison between GJson and json.UnMarshal for XLMetadata parsing for varying part sizes.Run using go test xl-meta-parse_test.go -run=xxx -bench=.

xl-meta-parse_test.go

/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"github.com/tidwall/gjson"
	"strconv"
	"testing"
	"time"
)

/* Benchamarks to compare the performance of parsing of XLMeta for growing number of parts.
   Json.UnMarshal and Gjson ("github.com/tidwall/gjson") functions are the compared for the speed of UnMarshalling.

   For this purpose Sample xl.sjon data is created for  parts sizes 10,100,1000,2000,5000,10000 part and then
   its parsed using the current implemntation of json.Unmarshal and the new approach Gjson library which is under scrutiny
   in the current benchmarks
*/

type objectPartInfo struct {
	Number int    `json:"number"`
	Name   string `json:"name"`
	ETag   string `json:"etag"`
	Size   int64  `json:"size"`
}

type checkSumInfo struct {
	Name      string `json:"name"`
	Algorithm string `json:"algorithm"`
	Hash      string `json:"hash"`
}

// Constant indicates current bit-rot algo used when creating objects.
const (
	bitRotAlgo = "blake2b"
)

// statInfo - carries stat information of the object.
type statInfo struct {
	Size    int64     `json:"size"`    // Size of the object `xl.json`.
	ModTime time.Time `json:"modTime"` // ModTime of the object `xl.json`.
}

type erasureInfo struct {
	Algorithm    string         `json:"algorithm"`
	DataBlocks   int            `json:"data"`
	ParityBlocks int            `json:"parity"`
	BlockSize    int64          `json:"blockSize"`
	Index        int            `json:"index"`
	Distribution []int          `json:"distribution"`
	Checksum     []checkSumInfo `json:"checksum,omitempty"`
}

// A xlMetaV1 represents `xl.json` metadata header.
type xlMetaV1 struct {
	Version string   `json:"version"` // Version of the current `xl.json`.
	Format  string   `json:"format"`  // Format of the current `xl.json`.
	Stat    statInfo `json:"stat"`    // Stat of the current object `xl.json`.
	// Erasure coded info for the current object `xl.json`.
	Erasure erasureInfo `json:"erasure"`
	// Minio release tag for current object `xl.json`.
	Minio struct {
		Release string `json:"release"`
	} `json:"minio"`
	// Metadata map for current object `xl.json`.
	Meta map[string]string `json:"meta,omitempty"`
	// Captures all the individual object `xl.json`.
	Parts []objectPartInfo `json:"parts,omitempty"`
}

// newXLMetaV1 - initializes new xlMetaV1, adds version, allocates a fresh erasure info.
func newXLMetaV1() xlMetaV1 {
	xlMeta := xlMetaV1{}
	xlMeta.Version = "1.0.0"
	xlMeta.Format = "xl"
	xlMeta.Minio.Release = "1.0.0"
	xlMeta.Erasure = erasureInfo{
		Algorithm:    "klauspost/reedsolomon/vandermonde",
		DataBlocks:   5,
		ParityBlocks: 5,
		BlockSize:    10485760,
		Distribution: []int{9, 10, 1, 2, 3, 4, 5, 6, 7, 8},
	}
	//
	return xlMeta
}

func (m *xlMetaV1) AddObjectCheckSum(checkSumNum int, name string, hash string, algo string) {
	checkSum := checkSumInfo{
		Name:      name,
		Algorithm: algo,
		Hash:      hash,
	}
	m.Erasure.Checksum[checkSumNum] = checkSum
}

// AddObjectPart - add a new object part in order.
func (m *xlMetaV1) AddObjectPart(partNumber int, partName string, partETag string, partSize int64) {
	partInfo := objectPartInfo{
		Number: partNumber,
		Name:   partName,
		ETag:   partETag,
		Size:   partSize,
	}

	// Proceed to include new part info.
	m.Parts[partNumber] = partInfo
}

func getXLMetaBytes(totalParts int) []byte {
	xlSampleMeta := getSampleXLMeta(totalParts)
	xlMetaBytes, err := json.Marshal(xlSampleMeta)
	if err != nil {
		panic(err)
	}
	return xlMetaBytes
}

func getSampleXLMeta(totalParts int) xlMetaV1 {
	xlMeta := newXLMetaV1()
	// Number of checksum info == total parts.
	xlMeta.Erasure.Checksum = make([]checkSumInfo, totalParts)
	// total number of parts.
	xlMeta.Parts = make([]objectPartInfo, totalParts)
	for i := 0; i < totalParts; i++ {
		partName := "part." + strconv.Itoa(i+1)
		// hard coding hash and algo value for the checksum, Since we are benchmarking the parsing of xl.json the magnitude doesn't affect the test,
		// The magnitude doesn't make a difference, only the size does.
		xlMeta.AddObjectCheckSum(i, partName, "a23f5eff248c4372badd9f3b2455a285cd4ca86c3d9a570b091d3fc5cd7ca6d9484bbea3f8c5d8d4f84daae96874419eda578fd736455334afbac2c924b3915a", "blake2b")
		xlMeta.AddObjectPart(i, partName, "d3fdd79cc3efd5fe5c068d7be397934b", 67108864)
	}
	return xlMeta
}

// Generates xlMetaV1 for given size, then benchmarks accessing the version field using the gjson library.
func gJSONBenchVersion(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaJson := getXLMetaBytes(totalParts)
	// Reset the benchmark timer, so that time spent on generating XLMeta is not considered.
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if gjson.Get(string(xlMetaJson), "version").Type == gjson.Null {
			b.Fatal("did not find the value")
		}
	}

}

// Generates xlMetaV1 for given size, then benchmarks accessing the version field using json.Unmarshal.
func unmarshalBenchVersion(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaJson := getXLMetaBytes(totalParts)
	// Reset the benchmark timer, so that time spent on generating XLMeta is not considered.
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var unMarshalXLMeta xlMetaV1
		if err := json.Unmarshal(xlMetaJson, &unMarshalXLMeta); err != nil {
			b.Fatal(err)
		}
		if unMarshalXLMeta.Version == "" {
			b.Fatalf("Version not fetched by Json unmarshalling")
		}

	}

}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 10 using GJSON library.
func BenchmarkGetVersionGJSON10(b *testing.B) {
	gJSONBenchVersion(b, 10)
}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 10 using json.Unmarshal.
func BenchmarkGetVersionUnMarshal10(b *testing.B) {
	unmarshalBenchVersion(b, 10)
}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 100 using GJSON library.
func BenchmarkGetVersionGJSON100(b *testing.B) {
	gJSONBenchVersion(b, 100)

}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 100 using json.Unmarshal.
func BenchmarkGetVersionUnMarshal100(b *testing.B) {
	unmarshalBenchVersion(b, 100)
}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 1000 using GJSON library.
func BenchmarkGetVersionGJSON1000(b *testing.B) {
	gJSONBenchVersion(b, 1000)
}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 1000 using json.Unmarshal.
func BenchmarkGetVersionUnMarshal1000(b *testing.B) {
	unmarshalBenchVersion(b, 1000)
}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 5000 using GJSON library.
func BenchmarkGetVersionGJSON5000(b *testing.B) {
	gJSONBenchVersion(b, 5000)

}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 5000 using json.Unmarshal.
func BenchmarkGetVersionUnMarshal5000(b *testing.B) {
	unmarshalBenchVersion(b, 5000)
}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 10000 using GJSON library.
func BenchmarkGetVersionGJSON10000(b *testing.B) {
	gJSONBenchVersion(b, 10000)

}

// Benchmark parsing of `version` field from the XLVMetaV1 json of part size 10000 using json.Unmarshal.
func BenchmarkGetVersionUnMarshal10000(b *testing.B) {
	unmarshalBenchVersion(b, 10000)
}

// Generates xlMetaV1 for given size,  then benchmarks the accessing of Checksum field of one of the parts info gjson library .
func gJSONBenchChecksumName(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaJson := getXLMetaBytes(totalParts)
	// Reset the benchmark timer, so that time spent on generating XLMeta is not considered.
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if gjson.Get(string(xlMetaJson), "erasure.checksum.5.name").Type == gjson.Null {
			b.Fatal("did not find the value")
		}
	}
}

// Generates xlMetaV1 for given size, then benchmarks accessing the Checksum field of one of the parts info using json.Unmarshal.
func unmarshalBenchChecksumName(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaJson := getXLMetaBytes(totalParts)
	// Reset the benchmark timer, so that time spent on generating XLMeta is not considered.
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var unMarshalXLMeta xlMetaV1
		if err := json.Unmarshal(xlMetaJson, &unMarshalXLMeta); err != nil {
			b.Fatal(err)
		}
		if unMarshalXLMeta.Erasure.Checksum[5].Name == "" {
			b.Fatalf("Version not fetched by Json unmarshalling")
		}

	}
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 10 using GJSON library.
func BenchmarkGetChecksumNameGJSON10(b *testing.B) {
	gJSONBenchChecksumName(b, 10)
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 10 using json.Unmarshal.
func BenchmarkGetChecksumNameUnMarshal10(b *testing.B) {
	unmarshalBenchChecksumName(b, 10)
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 100 using GJSON library.
func BenchmarkGetChecksumNameGJSON100(b *testing.B) {
	gJSONBenchChecksumName(b, 100)

}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 100 using json.Unmarshal.
func BenchmarkGetChecksumNameUnMarshal100(b *testing.B) {
	unmarshalBenchChecksumName(b, 100)
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 1000 using GJSON library.
func BenchmarkGetChecksumNameGJSON1000(b *testing.B) {
	gJSONBenchChecksumName(b, 1000)
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 1000 using json.Unmarshal.
func BenchmarkGetChecksumNameUnMarshal1000(b *testing.B) {
	unmarshalBenchChecksumName(b, 1000)
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 5000 using GJSON library.
func BenchmarkGetChecksumNameGJSON5000(b *testing.B) {
	gJSONBenchChecksumName(b, 5000)

}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 5000 using json.Unmarshal.
func BenchmarkGetChecksumNameUnMarshal5000(b *testing.B) {
	unmarshalBenchChecksumName(b, 5000)
}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 10000 using GJSON library.
func BenchmarkGetChecksumNameGJSON10000(b *testing.B) {
	gJSONBenchChecksumName(b, 10000)

}

// Benchmark parsing of checksum field of one of the parts info from the XLVMetaV1 json of part size 10000 using json.Unmarshal.
func BenchmarkGetChecksumNameUnMarshal10000(b *testing.B) {
	unmarshalBenchChecksumName(b, 10000)
}

// Generates xlMetaV1 for given size,  then benchmarks the accessing of the parts info field using gjson library .
func gJSONBenchPartsInfo(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaJson := getXLMetaBytes(totalParts)
	// Reset the benchmark timer, so that time spent on generating XLMeta is not considered.
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if gjson.Get(string(xlMetaJson), "parts.5.name").Type == gjson.Null {
			b.Fatal("Did not find parts value")
		}
	}
}

// Generates xlMetaV1 for given size, then benchmarks the accessing of the parts info field using json.Unmarshal.
func unmarshalBenchPartsInfo(b *testing.B, totalParts int) {
	// obtain XLMetaV1 for part size `totalParts`.
	xlMetaJson := getXLMetaBytes(totalParts)
	// Reset the benchmark timer, so that time spent on generating XLMeta is not considered.
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		var unMarshalXLMeta xlMetaV1
		if err := json.Unmarshal(xlMetaJson, &unMarshalXLMeta); err != nil {
			b.Fatal(err)
		}
		if unMarshalXLMeta.Parts[5].Name == "" {
			b.Fatalf("Parts not fetched by Json unmarshalling")
		}

	}
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 10 using GJSON library.
func BenchmarkGetPartsInfoGJSON10(b *testing.B) {
	gJSONBenchPartsInfo(b, 10)
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 10 using json.Unmarshal.
func BenchmarkGetPartsInfoUnMarshal10(b *testing.B) {
	unmarshalBenchPartsInfo(b, 10)
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 100 using GJSON library.
func BenchmarkGetPartsInfoGJSON100(b *testing.B) {
	gJSONBenchPartsInfo(b, 100)

}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 100 using json.Unmarshal.
func BenchmarkGetPartsInfoUnMarshal100(b *testing.B) {
	unmarshalBenchPartsInfo(b, 100)
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 1000 using GJSON library.
func BenchmarkGetPartsInfoGJSON1000(b *testing.B) {
	gJSONBenchPartsInfo(b, 1000)
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 1000 using json.Unmarshal.
func BenchmarkGetPartsInfoUnMarshal1000(b *testing.B) {
	unmarshalBenchPartsInfo(b, 1000)
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 5000 using GJSON library.
func BenchmarkGetPartsInfoGJSON5000(b *testing.B) {
	gJSONBenchPartsInfo(b, 5000)

}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 5000 using json.Unmarshal.
func BenchmarkGetPartsInfoUnMarshal5000(b *testing.B) {
	unmarshalBenchPartsInfo(b, 5000)
}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 10000 using GJSON library.
func BenchmarkGetPartsInfoGJSON10000(b *testing.B) {
	gJSONBenchPartsInfo(b, 10000)

}

// Benchmark parsing of parts info from the XLVMetaV1 json of part size 10000 using json.Unmarshal.
func BenchmarkGetPartsInfoUnMarshal10000(b *testing.B) {
	unmarshalBenchPartsInfo(b, 10000)
}