Skip to content

Instantly share code, notes, and snippets.

@saran2020
Last active December 28, 2024 16:47
Show Gist options
  • Save saran2020/c2b826b26d83cff3f320c5c60dffd4e2 to your computer and use it in GitHub Desktop.
Save saran2020/c2b826b26d83cff3f320c5c60dffd4e2 to your computer and use it in GitHub Desktop.
Expirment to generate JSON and Proto data and then gzip them to compare the results.
package main
import (
"bufio"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"math/rand/v2"
"os"
"text/tabwriter"
"google.golang.org/protobuf/proto"
)
const (
usernamesFile = "usernames.txt"
)
var UserCounts = [...]int{1, 10, 100, 1000, 10000, 100000, 1000000}
func main() {
names := readUsername(usernamesFile)
usersList := UsersProto{
Users: []*UserProto{},
}
writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.Debug)
fmt.Fprintln(writer, "Users\tJSON Size\tGzipped JSON Size\tJSON Gzip size % \tProto Size\tGzipped Proto Size\tProto Gzip size %\tGzip Diff (JSON - Proto)")
for _, num := range UserCounts {
for i := 0; i < num; i++ {
name := names[rand.IntN(len(names))]
user := UserProto{
Name: name,
Age: rand.Int32N(91) + 10,
Email: fmt.Sprintf("%s@gmail.com", name),
}
usersList.Users = append(usersList.Users, &user)
}
jsonData, _ := json.Marshal(usersList)
protData, _ := proto.Marshal(&usersList)
gzippedProtoSize := gzipDataAndReturnSize(protData)
gzippedJsonSize := gzipDataAndReturnSize(jsonData)
jsonReduction := float64(gzippedJsonSize) / float64(len(jsonData)) * 100
protoReduction := float64(gzippedProtoSize) / float64(len(protData)) * 100
diff := gzippedJsonSize - gzippedProtoSize
fmt.Fprintf(writer, "%d\t%s\t%s\t%.0f%%\t%s\t%s\t%.0f%%\t%s\n",
num,
humanReadableSize(len(jsonData)),
humanReadableSize(gzippedJsonSize),
jsonReduction,
humanReadableSize(len(protData)),
humanReadableSize(gzippedProtoSize),
protoReduction,
humanReadableSize(diff),
)
}
writer.Flush()
}
// humanReadableSize returns a human-readable size string.
// e.g. 1024 -> 1 KB
// e.g. 1048576 -> 1 MB
func humanReadableSize(bytes int) string {
const unit = 1024
absBytes := abs(bytes)
if absBytes < unit {
return fmt.Sprintf("%dB", absBytes)
}
div, exp := int64(unit), 0
for n := absBytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f%cB", float64(signOf(bytes, absBytes))/float64(div), "KMGTPE"[exp])
}
// abs returns the absolute value of x.
func abs(x int) int {
if x < 0 {
return -x
}
return x
}
// signOf returns the sign of the original number applied to the applySignTo number.
func signOf(original int, applySignTo int) int {
if (original < 0) == (applySignTo < 0) {
return applySignTo
}
return -applySignTo
}
func readUsername(fileName string) []string {
file, _ := os.Open(fileName)
defer file.Close()
scanner := bufio.NewScanner(file)
var names []string
for scanner.Scan() {
names = append(names, scanner.Text())
}
return names
}
func gzipDataAndReturnSize(data []byte) int {
var buf bytes.Buffer
gw := gzip.NewWriter(&buf)
gw.Write(data)
gw.Close()
return buf.Len()
}
Users |JSON Size |Gzipped JSON Size |JSON Gzip size % |Proto Size |Gzipped Proto Size |Proto Gzip size % |Gzip Diff (JSON - Proto)
1 |75B |85B |113% |40B |55B |138% |30B
10 |673B |232B |34% |398B |216B |54% |16B
100 |6.3KB |1.4KB |22% |3.7KB |1.4KB |37% |12B
1000 |62.5KB |11.8KB |19% |36.5KB |12.3KB |34% |542B
10000 |624.9KB |115.7KB |19% |364.3KB |121.7KB |33% |-6.0KB
100000 |6.1MB |1.1MB |18% |3.6MB |1.2MB |33% |-60.7KB
1000000 |61.1MB |11.3MB |18% |35.7MB |11.9MB |33% |-603.8KB
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment