box.execute() measurements

.gitignore

tarantool-[abc]
log.txt
*.snap
*.xlog
*.csv

README.md

Overview

Benchmark box.execute() using SELECT 1 queries to verify #4231 implementations.

Experiment set up

Clone tarantool into three directories: tarantool-a, tarantool-b, tarantool-c.

tarantool-a

Tarantool 2.5.0-16-g9f2819785.

tarantool-b

The same as previous, but with Maria's patches from eljashm/gh-4231-box.execute-immutable-function upward (9383e5683c4df9bc9d3b04a2c0bf945b9dcb3bc1 and e2648197b0bb580a1d06033e5dff853b6b4a4569).

tarantool-c

The same as previous, but with caching of the value in Lua:

diff --git a/src/box/lua/load_cfg.lua b/src/box/lua/load_cfg.lua
index 3ab43b000..de70c5c63 100644
--- a/src/box/lua/load_cfg.lua
+++ b/src/box/lua/load_cfg.lua
@@ -556,8 +556,18 @@ setmetatable(box, {
   end
 })

+local box_is_configured_cached = false
 local function box_is_configured()
-    return ffi.C.box_is_configured()
+    -- Using of cached Lua value is faster then ffi call.
+    if box_is_configured_cached then
+        return true
+    end
+
+    local res = ffi.C.box_is_configured()
+    if res == true then
+        box_is_configured_cached = true
+    end
+    return res
 end

 local function load_cfg(cfg)

How to build

$ cd tarantool-a
$ cmake .                             \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DENABLE_BACKTRACE=ON             \
    -DENABLE_DIST=ON                  \
    -DENABLE_BUNDLED_LIBCURL=OFF      \
    -DENABLE_WERROR=ON                \
    && make -j

How to collect results

$ ./run.sh

Results will be saved into result_*.txt files.

Then figures can be plotted:

$ ./gen_csv.sh
$ gnuplot plot_avg.txt
$ gnuplot plot.txt

bench.lua

#!/usr/bin/env tarantool

local clock = require('clock')

box.cfg{log = 'log.txt'}

local start_time = clock.monotonic()
for i = 1, 10^6 do
    box.execute('SELECT 1')
end
local duration = clock.monotonic() - start_time
print(duration)

os.exit()

gen_csv.sh

#!/bin/sh

for var in a b c; do
    for i in $(seq 1 10); do
        result_file="result_${var}_${i}.txt"
        j=0
        while read line; do
            j=$((j+1))
            n=$((j+10*(i-1)))
            printf "${n},${line}\n"
        done < "${result_file}"
    done > "result_${var}.csv"
done

for var in a b c; do
    for i in $(seq 1 10); do
        result_file="result_${var}_${i}.txt"
        ./stat.awk < "${result_file}" | grep '^avg:' | awk '{ print "'"${i}"',"$2; }'
    done > result_avg_${var}.csv
done

plot.svg

plot.txt

set terminal svg enhanced linewidth 0.5 font 'Verdana,5' size 800,400
set output 'plot.svg'

set datafile separator ","

set bmargin 5

plot 'result_a.csv' with lines, \
    'result_b.csv' with lines, \
    'result_c.csv' with lines

plot_avg.svg

plot_avg.txt

set terminal svg enhanced linewidth 0.5 font 'Verdana,5' size 800,400
set output 'plot_avg.svg'

set datafile separator ","

set bmargin 5

plot 'result_avg_a.csv' with lines, \
    'result_avg_b.csv' with lines, \
    'result_avg_c.csv' with lines

result_a_1.txt

2.183851675014
2.2010253299959
2.2059130580164
2.1548372660764
2.1981472680345
2.1964296519291
2.1530008020345
2.203642737004
2.1902553950204
2.2068677869393

result_a_10.txt

2.4238469790434
2.3857251110021
2.3726613880135
2.3562455849024
2.3547751719598
2.3390066369902
2.3709119899431
2.3635417009937
2.3670372019988
2.3617378970375

result_a_2.txt

2.2431526610162
2.2249788320623
2.2233266300755
2.3686036400031
2.3968613180332
2.3051012769574
2.2699106959626
2.2742626300314
2.3118613419356
2.2421325709438

result_a_3.txt

2.3216244590003
2.2970892440062
2.2774457340129
2.308553775074
2.3199673019117
2.2862964730011
2.3430129119661
2.30376385001
2.3088064539479
2.3143525889609

result_a_4.txt

2.3233553649625
2.3000423880294
2.2956064089667
2.3823762529064
2.3148005449912
2.3151135300286
2.2770598960342
2.3510061210254
2.3037679110421
2.3336005699821

result_a_5.txt

2.2778507500188
2.2870792039903
2.3026154020336
2.3273941009538
2.3238671029685
2.3444810310612
2.287918947055
2.3059889629949
2.2955966240261
2.3441700520925

result_a_6.txt

2.3140660169302
2.3302840030519
2.3209050010191
2.3518842019839
2.34396985895
2.3253920850111
2.3019648719346
2.3279942299705
2.3386879099999
2.3324496719288

result_a_7.txt

2.3416609120322
2.3784086280502
2.319693802041
2.3515759140719
2.5320774890715
2.3334857169539
2.3566832549404
2.3432298999978
2.3501163759502
2.3657067290042

result_a_8.txt

2.3837763059419
2.360636768979
2.3292783709476
2.3109147050418
2.3435845959466
2.3728822550038
2.3554537569871
2.3564220459666
2.3998506349744
2.341564183007

result_a_9.txt

2.3663268119562
2.3673711690353
2.3759915729752
2.3409229828976
2.3811626080424
2.3540755830472
2.3436625490431
2.3701225990662
2.3626037570648
2.3625621440588

result_b_1.txt

2.1721468300093
2.2157173969317
2.3309524039505
2.3213202729821
2.2041695190128
2.1503148599295
2.2143290729728
2.1809951791074
2.2159951870563
2.1970973620191

result_b_10.txt

2.3476838479983
2.3698919869494
2.4778261779575
2.3651461750269
2.3496906630462
2.3912773960037
2.3335878551006
2.3617814480094
2.3539334990783
2.3780213490827

result_b_2.txt

2.2398517410038
2.2698501150589
2.2624415040482
2.2667167090112
2.2486715570558
2.2581154530635
2.288227356039
2.2673271680251
2.2595223239623
2.3006926330272

result_b_3.txt

2.3011557710124
2.3122267080471
2.3125644159736
2.315487607033
2.3187415889697
2.3406896529486
2.3534616389079
2.3285531690344
2.3727233769605
2.323025865946

result_b_4.txt

2.2946393380407
2.3409075349336
2.3659313800745
2.3489935430698
2.2989957339596
2.2925689490512
2.3091021151049
2.3077802379848
2.3305225620279
2.321394560975

result_b_5.txt

2.3001921509858
2.3555914819008
2.2960486439988
2.3028342280304
2.319501243066
2.302100069006
2.2963371230289
2.2947819929104
2.3028026969405
2.2893871699926

result_b_6.txt

2.3749364509713
2.3642615820281
2.3343368470669
2.3546078240033
2.3504951969953
2.3229639199562
2.3644168049796
2.3312463210896
2.3277274060529
2.3465569580439

result_b_7.txt

2.3287260549841
2.3752686119406
2.3354461489944
2.3639543890022
2.3480944569455
2.390899463906
2.3549479009816
2.315207852982
2.3507516619284
2.3530765089672

result_b_8.txt

2.3736338580493
2.3712129920023
2.4255959839793
2.3598586380249
2.3627250340069
2.3381837169873
2.3626922300318
2.3688410969917
2.3619190879399
2.3500282989116

result_b_9.txt

2.3761640820885
2.3638811931014
2.4033743360778
2.3469399219612
2.3692381230649
2.3584292650921
2.3595145659056
2.381172241061
2.351987412083
2.3827444280032

result_c_1.txt

2.2262576729991
2.2116516559618
2.2331773809856
2.225227690069
2.2231981629739
2.2320844889618
2.2474297389854
2.2337648599641
2.2324356409954
2.2738552379888

result_c_10.txt

2.3579911000561
2.3682610290125
2.3679134659469
2.5600304179825
2.4205272969557
2.3566284120316
2.3842005199986
2.3420283310115
2.3674776169937
2.3530219569802

result_c_2.txt

2.4299559619976
2.2710726889782
2.2668603280326
2.2813214350026
2.2853040259797
2.307692958042
2.288833115017
2.2896101720398
2.2800726820715
2.265849368996

result_c_3.txt

2.3019344470231
2.3073219190119
2.32897284301
2.4311254660133
2.3270523620304
2.2977166129276
2.292844603071
2.3474997379817
2.3157216929831
2.291511913063

result_c_4.txt

2.3040684879525
2.3410955670988
2.3151469100267
2.3125061660539
2.2995461549144
2.3134762439877
2.3134062360041
2.2926461489405
2.3042298349319
2.2706114730099

result_c_5.txt

2.3084959780099
2.3205928140087
2.3058324039448
2.2998834209284
2.3147991379956
2.3079335809452
2.3597684049746
2.3573643629206
2.3249161699787
2.3214313089848

result_c_6.txt

2.3618965820642
2.3294959800551
2.3512242659926
2.3327103459742
2.3690591719933
2.3706771460129
2.3385903210146
2.3287489770446
2.3347806980601
2.3331263510045

result_c_7.txt

2.3727868050337
2.3430765660014
2.3544429650065
2.3367907009088
2.3293382209959
2.3758684400236
2.3561903620139
2.3101701360429
2.361468355055
2.3448743249755

result_c_8.txt

2.4091879209736
2.3856923410203
2.3955610980047
2.3622817189898
2.3553720399505
2.3751690589124
2.3330018620472
2.3613747929921
2.389731174917
2.3323583758902

result_c_9.txt

2.3494517450454
2.3297445769422
2.3839998419862
2.3541212070268
2.3690287499921
2.3676315339981
2.3986981040798
2.3974225240527
2.3616663200082
2.3690825259546

run.sh

#!/bin/sh

for i in $(seq 1 10); do
    for var in a b c; do
        result_file="result_${var}_${i}.txt"
        for j in $(seq 1 10); do
            rm -f *.{xlog,snap}
            taskset 0x2 ./tarantool-${var}/src/tarantool bench.lua >> "${result_file}"
        done
    done
done

stat.awk

#!/usr/bin/awk -f

BEGIN {
    min = "NaN"
    max = "NaN"
    cnt = 0
    var = "NaN"
    sum = "NaN"
    avg = "NaN"
}
{
    min = (NR==1 || $1<min ? $1 : min)
    max = (NR==1 || $1>max ? $1 : max)
    cnt += 1
    sum += $1
}
END {
    avg = sum / cnt
    var = max - min
    var_rel = 100 * var / avg
    printf "cnt: %d\n", cnt
    printf "min: %.6f\n", min
    printf "max: %.6f\n", max
    printf "avg: %.6f\n", avg
    printf "var: %.6f (%.6f%)\n", var, var_rel
}