-
-
Save saran2020/c2b826b26d83cff3f320c5c60dffd4e2 to your computer and use it in GitHub Desktop.
Expirment to generate JSON and Proto data and then gzip them to compare the results.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"bufio" | |
"bytes" | |
"compress/gzip" | |
"encoding/json" | |
"fmt" | |
"math/rand/v2" | |
"os" | |
"text/tabwriter" | |
"google.golang.org/protobuf/proto" | |
) | |
const ( | |
usernamesFile = "usernames.txt" | |
) | |
var UserCounts = [...]int{1, 10, 100, 1000, 10000, 100000, 1000000} | |
func main() { | |
names := readUsername(usernamesFile) | |
usersList := UsersProto{ | |
Users: []*UserProto{}, | |
} | |
writer := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.Debug) | |
fmt.Fprintln(writer, "Users\tJSON Size\tGzipped JSON Size\tJSON Gzip size % \tProto Size\tGzipped Proto Size\tProto Gzip size %\tGzip Diff (JSON - Proto)") | |
for _, num := range UserCounts { | |
for i := 0; i < num; i++ { | |
name := names[rand.IntN(len(names))] | |
user := UserProto{ | |
Name: name, | |
Age: rand.Int32N(91) + 10, | |
Email: fmt.Sprintf("%s@gmail.com", name), | |
} | |
usersList.Users = append(usersList.Users, &user) | |
} | |
jsonData, _ := json.Marshal(usersList) | |
protData, _ := proto.Marshal(&usersList) | |
gzippedProtoSize := gzipDataAndReturnSize(protData) | |
gzippedJsonSize := gzipDataAndReturnSize(jsonData) | |
jsonReduction := float64(gzippedJsonSize) / float64(len(jsonData)) * 100 | |
protoReduction := float64(gzippedProtoSize) / float64(len(protData)) * 100 | |
diff := gzippedJsonSize - gzippedProtoSize | |
fmt.Fprintf(writer, "%d\t%s\t%s\t%.0f%%\t%s\t%s\t%.0f%%\t%s\n", | |
num, | |
humanReadableSize(len(jsonData)), | |
humanReadableSize(gzippedJsonSize), | |
jsonReduction, | |
humanReadableSize(len(protData)), | |
humanReadableSize(gzippedProtoSize), | |
protoReduction, | |
humanReadableSize(diff), | |
) | |
} | |
writer.Flush() | |
} | |
// humanReadableSize returns a human-readable size string. | |
// e.g. 1024 -> 1 KB | |
// e.g. 1048576 -> 1 MB | |
func humanReadableSize(bytes int) string { | |
const unit = 1024 | |
absBytes := abs(bytes) | |
if absBytes < unit { | |
return fmt.Sprintf("%dB", absBytes) | |
} | |
div, exp := int64(unit), 0 | |
for n := absBytes / unit; n >= unit; n /= unit { | |
div *= unit | |
exp++ | |
} | |
return fmt.Sprintf("%.1f%cB", float64(signOf(bytes, absBytes))/float64(div), "KMGTPE"[exp]) | |
} | |
// abs returns the absolute value of x. | |
func abs(x int) int { | |
if x < 0 { | |
return -x | |
} | |
return x | |
} | |
// signOf returns the sign of the original number applied to the applySignTo number. | |
func signOf(original int, applySignTo int) int { | |
if (original < 0) == (applySignTo < 0) { | |
return applySignTo | |
} | |
return -applySignTo | |
} | |
func readUsername(fileName string) []string { | |
file, _ := os.Open(fileName) | |
defer file.Close() | |
scanner := bufio.NewScanner(file) | |
var names []string | |
for scanner.Scan() { | |
names = append(names, scanner.Text()) | |
} | |
return names | |
} | |
func gzipDataAndReturnSize(data []byte) int { | |
var buf bytes.Buffer | |
gw := gzip.NewWriter(&buf) | |
gw.Write(data) | |
gw.Close() | |
return buf.Len() | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Users |JSON Size |Gzipped JSON Size |JSON Gzip size % |Proto Size |Gzipped Proto Size |Proto Gzip size % |Gzip Diff (JSON - Proto) | |
1 |75B |85B |113% |40B |55B |138% |30B | |
10 |673B |232B |34% |398B |216B |54% |16B | |
100 |6.3KB |1.4KB |22% |3.7KB |1.4KB |37% |12B | |
1000 |62.5KB |11.8KB |19% |36.5KB |12.3KB |34% |542B | |
10000 |624.9KB |115.7KB |19% |364.3KB |121.7KB |33% |-6.0KB | |
100000 |6.1MB |1.1MB |18% |3.6MB |1.2MB |33% |-60.7KB | |
1000000 |61.1MB |11.3MB |18% |35.7MB |11.9MB |33% |-603.8KB |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment