Skip to content

Instantly share code, notes, and snippets.

@CertainLach
Last active July 10, 2022 12:00
Show Gist options
  • Save CertainLach/5770d7ad4836066f8e0bd91e823e451b to your computer and use it in GitHub Desktop.
Save CertainLach/5770d7ad4836066f8e0bd91e823e451b to your computer and use it in GitHub Desktop.
Jrsonnet performance

Benchmark results

large_string_join

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet large_string_join.jsonnet 16.5 ± 0.5 16.1 19.6 1.00
gojsonnet large_string_join.jsonnet 90.7 ± 3.8 85.6 104.0 5.49 ± 0.29
jsonnet large_string_join.jsonnet 69.5 ± 2.4 68.2 78.1 4.20 ± 0.20
sjsonnet large_string_join.jsonnet 760.6 ± 12.7 739.9 787.2 46.00 ± 1.67

large_string_template

Golang overflows OS stack on this test

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet large_string_template.jsonnet 7.8 ± 0.2 7.6 10.0 1.00
jsonnet large_string_template.jsonnet 18415.0 ± 39.4 18371.7 18478.6 2345.88 ± 74.55
sjsonnet large_string_template.jsonnet 612.5 ± 7.7 602.6 628.2 78.03 ± 2.66

realistic1

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet realistic1.jsonnet 47.1 ± 0.7 46.5 49.7 1.00
gojsonnet realistic1.jsonnet 8256.5 ± 41.0 8208.7 8333.1 175.25 ± 2.68
jsonnet realistic1.jsonnet 25659.5 ± 229.7 25434.8 26115.0 544.64 ± 9.26
sjsonnet realistic1.jsonnet 837.8 ± 18.8 809.6 866.0 17.78 ± 0.47

realistic2

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet realistic2.jsonnet 476.6 ± 5.0 472.5 489.8 1.00
gojsonnet realistic2.jsonnet 6890.0 ± 112.4 6808.3 7198.1 14.46 ± 0.28
jsonnet realistic2.jsonnet 31598.2 ± 645.3 30772.8 33067.6 66.30 ± 1.52
sjsonnet realistic2.jsonnet 1997.3 ± 48.1 1917.2 2053.5 4.19 ± 0.11

bench.01

Scala impl fails with obscure exception in this test

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.01.jsonnet 1.7 ± 0.3 1.5 3.4 1.00
gojsonnet bench.01.jsonnet 4.6 ± 0.4 4.3 6.5 2.64 ± 0.46
jsonnet bench.01.jsonnet 16.4 ± 1.3 15.8 27.7 9.37 ± 1.64

bench.02

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.02.jsonnet 1.4 ± 0.1 1.3 2.4 1.00
gojsonnet bench.02.jsonnet 984.9 ± 23.1 960.0 1021.1 715.07 ± 46.26
jsonnet bench.02.jsonnet 2673.7 ± 93.9 2550.2 2836.6 1941.17 ± 135.48
sjsonnet bench.02.jsonnet 851.7 ± 48.1 784.8 931.2 618.34 ± 51.10

bench.03

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.03.jsonnet 127.4 ± 3.2 121.7 132.1 1.00
gojsonnet bench.03.jsonnet 394.7 ± 14.8 374.9 420.2 3.10 ± 0.14
jsonnet bench.03.jsonnet 163.0 ± 6.9 154.8 175.1 1.28 ± 0.06
sjsonnet bench.03.jsonnet 891.3 ± 33.5 843.8 951.1 7.00 ± 0.31

bench.04

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.04.jsonnet 173.7 ± 6.9 162.4 182.8 1.00
gojsonnet bench.04.jsonnet 417.7 ± 21.2 390.0 454.3 2.40 ± 0.16
jsonnet bench.04.jsonnet 1253.5 ± 54.0 1205.7 1348.8 7.22 ± 0.42
sjsonnet bench.04.jsonnet 1223.4 ± 147.7 1014.8 1458.6 7.04 ± 0.90

bench.05.gen

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.05.gen.jsonnet 690.3 ± 21.5 660.8 736.7 1.00
gojsonnet bench.05.gen.jsonnet 1367.8 ± 22.2 1337.9 1395.6 1.98 ± 0.07
jsonnet bench.05.gen.jsonnet 1880.8 ± 42.2 1823.0 1931.6 2.72 ± 0.10
sjsonnet bench.05.gen.jsonnet 868.7 ± 29.8 817.7 907.6 1.26 ± 0.06

bench.06

std.reverse is not implemented in scala

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.06.jsonnet 3.6 ± 0.4 3.1 5.6 1.00
gojsonnet bench.06.jsonnet 12.8 ± 1.2 11.1 18.9 3.58 ± 0.49
jsonnet bench.06.jsonnet 3951.9 ± 148.4 3799.0 4214.1 1109.22 ± 121.41

bench.07

C++ and golang impl overflows stack in this test with default settings Scala impl fails with stack overflow in this test

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.07.jsonnet 1.7 ± 0.1 1.6 2.8 1.00
gojsonnet -s 200000 bench.07.jsonnet 715.1 ± 14.6 697.9 734.6 423.13 ± 30.66
jsonnet -s 200000 bench.07.jsonnet 167.1 ± 2.9 165.1 175.6 98.89 ± 7.08

bench.08

Command Mean [ms] Min [ms] Max [ms] Relative
jrsonnet bench.08.jsonnet 1.3 ± 0.1 1.2 2.3 1.00
gojsonnet bench.08.jsonnet 1.6 ± 0.2 1.4 2.8 1.17 ± 0.18
jsonnet bench.08.jsonnet 1309.8 ± 40.9 1282.4 1415.8 971.10 ± 80.36
sjsonnet bench.08.jsonnet 498.5 ± 10.5 485.8 514.1 369.56 ± 29.37
# Nix flake, which generates document above
{
description = "Jrsonnet benchmark";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
inputs.flake-utils.url = "github:numtide/flake-utils";
inputs.jrsonnet.url = "github:CertainLach/jrsonnet";
outputs = { self, nixpkgs, flake-utils, jrsonnet }:
flake-utils.lib.eachDefaultSystem (system:
let pkgs = nixpkgs.legacyPackages.${system};
in {
defaultPackage = pkgs.stdenv.mkDerivation {
name = "jrsonnet-benchmark";
buildInputs = [
jrsonnet.outputs.defaultPackage.${system}
pkgs.rustfmt
pkgs.cargo
pkgs.hyperfine
pkgs.jsonnet
(pkgs.callPackage pkgs.stdenv.mkDerivation {
name = "sjsonnet";
src = pkgs.fetchurl {
url =
"https://github.com/databricks/sjsonnet/releases/download/0.2.6/sjsonnet.jar";
sha256 = "076fgpj1kmqix157c8cqz617z6r99nwmy125p7gpi8nnrxc90bvs";
};
unpackPhase = "true";
buildInputs =
[ pkgs.adoptopenjdk-hotspot-bin-8 pkgs.makeWrapper ];
installPhase = ''
mkdir -p $out/bin $out/lib
cp $src $out/lib/sjsonner.jar
makeWrapper ${pkgs.adoptopenjdk-hotspot-bin-8}/bin/java $out/bin/sjsonnet --add-flags "-jar $out/lib/sjsonner.jar"
'';
separateDebugInfo = false;
})
(pkgs.callPackage pkgs.stdenv.mkDerivation {
name = "gojsonnet";
unpackPhase = "true";
buildInputs = [ pkgs.go-jsonnet ];
installPhase = ''
mkdir -p $out/bin
echo "Test"
cp ${pkgs.go-jsonnet}/bin/jsonnet $out/bin/gojsonnet
'';
separateDebugInfo = false;
})
];
src = pkgs.fetchFromGitHub {
owner = "google";
repo = "jsonnet";
rev = "2991580ddae4027384502cfe11a31bd032d58247";
sha256 = "15csk6bhvjkjy3fg3g06pg5pds41lgfrgnxr435s8cyy6m1bjr23";
};
buildPhase = "true";
installPhase = let
bench = name:
{ go ? true, scala ? true, cpp ? true, large ? false
, comment ? null }:
let path = "${name}.jsonnet";
in ''
hyperfine --style basic --export-markdown 'result.md' 'jrsonnet ${path}' ${
if go then
"'gojsonnet ${if large then "-s 200000" else ""} ${path}'"
else
""
} ${
if cpp then
"'jsonnet ${if large then "-s 200000" else ""} ${path}'"
else
""
} ${if scala then "'sjsonnet ${path}'" else ""}
echo '## ${name}' >> $out
${if comment != null then ''
echo >> $out
echo "${comment}" >> $out
'' else ''
echo >> $out
''}
cat result.md >> $out
echo >> $out
'';
in ''
echo "# Benchmark results" > $out
echo "" >> $out
cd perf_tests
${bench "large_string_join" { }}
${bench "large_string_template" {
go = false;
comment = ''
> Golang overflows OS stack on this test
'';
}}
${bench "realistic1" { }}
${bench "realistic2" { }}
cd ../benchmarks
jrsonnet -S gen_big_object.jsonnet > bench.05.gen.jsonnet
${bench "bench.01" {
scala = false;
comment = ''
> Scala impl fails with obscure exception in this test
'';
}}
${bench "bench.02" { }}
${bench "bench.03" { }}
${bench "bench.04" { }}
${bench "bench.05.gen" { }}
${bench "bench.06" {
scala = false;
comment = ''
> std.reverse is not implemented in scala
'';
}}
${bench "bench.07" {
large = true;
scala = false;
comment = ''
> C++ and golang impl overflows stack in this test with default settings
> Scala impl fails with stack overflow in this test
'';
}}
${bench "bench.08" { }}
'';
};
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment