gistfile1.txt

cached test
server use discard (SSD TRIM)

measured from iostat

for dyn in 0 1; do for cpri in 0 3; do for j in 1 2 4 8 12 16 24 32 40 48 64 ; do echo j $j, pri $cpri, dyn $dyn; head -3 hw.j$j.pri$cpri.dyn$dyn | tail -1 ; done; done; done | grep -v pri | awk '{ printf "%.0f\t%.0f\t%.0f\t%.0f\n", $3, $5, $6, $7 }'
rMB/s   wMB/s   rGB     wGB
0       210     0       2075
0       258     0       2083
0       647     0       1773
0       1091    0       1716
0       1264    0       1781
0       1431    0       1797
0       1500    0       1752
0       1498    0       1720
0       1504    0       1706
0       1523    0       1675
0       1559    0       1701
0       198     0       1423
0       261     0       1426
0       605     0       1260
0       902     0       1333
0       1066    0       1395
0       1153    0       1414
0       1266    0       1435
0       1286    0       1440
0       1333    0       1455
0       1357    0       1477
0       1332    0       1434
0       214     0       2129
0       256     0       2081
0       657     0       1869
0       1162    0       1872
0       1387    0       1839
0       1486    0       1858
0       1528    0       1780
0       1530    0       1772
0       1531    0       1712
0       1527    0       1709
0       1644    0       1776
0       196     0       1374
0       260     0       1376
0       624     0       1308
0       966     0       1402
0       1146    0       1453
0       1218    0       1465
0       1313    0       1489
0       1366    0       1507
0       1429    0       1528
0       1447    0       1526
0       1424    0       1491

measured from compaction IO stats
* wGB.1, rGB.2 - GB for writes, reads during fillrandom
* wGB.2, rGB.2 - GB for writes, reads while waiting for compaction to finish
* wGB.t - wGB.1 + wGB.2
* rGB.t - rGB.1 + rGB.2

for dyn in 0 1; do for cpri in 0 3; do for j in 1 2 4 8 12 16 24 32 40 48 64 ; do echo j $j, pri $cpri, dyn $dyn; tail -1 hw.j$j.pri$cpri.dyn$dyn | tail -1 ; done; done; done | grep -v pri 
wGB.1   rGB.1 wGB.2   rGB.2    wGB.t   rGB.t
798.39 57.87 758.02 63.37 :: 856.26 821.39
788.48 52.61 750.65 55.62 :: 841.09 806.27
645.06 53.57 607.24 56.52 :: 698.63 663.76
623.53 49.54 585.44 52.40 :: 673.07 637.84
667.25 33.83 629.50 35.76 :: 701.08 665.26
674.17 33.06 636.75 34.43 :: 707.23 671.18
658.74 27.73 621.69 28.94 :: 686.47 650.63
642.78 29.75 604.87 31.25 :: 672.53 636.12
632.12 32.14 594.10 33.57 :: 664.26 627.67
627.55 24.47 589.70 25.69 :: 652.02 615.39
630.75 32.39 592.46 33.75 :: 663.14 626.21
478.13 47.97 436.54 53.68 :: 526.10 490.22
481.01 47.31 439.40 53.06 :: 528.32 492.46
410.59 46.48 369.85 51.17 :: 457.07 421.02
460.22 31.87 420.48 34.29 :: 492.09 454.77
496.42 24.25 457.50 25.80 :: 520.67 483.30
508.53 21.09 469.76 22.31 :: 529.62 492.07
524.29 16.90 485.97 17.72 :: 541.19 503.69
523.99 19.04 485.58 20.10 :: 543.03 505.68
531.24 18.79 492.88 19.78 :: 550.03 512.66
541.68 17.85 503.36 18.75 :: 559.53 522.11
519.27 19.89 480.65 20.69 :: 539.16 501.34
806.81 58.27 769.23 61.07 :: 865.08 830.30
800.31 43.57 762.72 46.32 :: 843.88 809.04
697.39 46.25 659.38 49.15 :: 743.64 708.53
694.97 49.29 656.82 52.26 :: 744.26 709.08
684.76 42.45 647.42 44.37 :: 727.21 691.79
697.52 39.68 660.30 41.22 :: 737.20 701.52
673.57 28.61 636.15 30.08 :: 702.18 666.23
671.00 25.84 633.89 26.73 :: 696.84 660.62
642.99 27.62 605.42 29.09 :: 670.61 634.51
643.02 26.33 605.73 27.56 :: 669.35 633.29
683.24 15.98 646.06 16.50 :: 699.22 662.56
455.07 48.37 413.39 54.03 :: 503.44 467.42
459.79 45.52 418.31 51.03 :: 505.31 469.34
440.86 38.11 400.77 41.94 :: 478.97 442.71
494.85 29.76 456.33 31.86 :: 524.61 488.19
523.73 23.00 485.74 24.58 :: 546.73 510.32
531.14 20.28 493.42 21.58 :: 551.42 515.00
547.68 16.23 510.24 17.24 :: 563.91 527.48
556.64 16.05 519.12 17.05 :: 572.69 536.17
562.75 18.99 525.14 20.17 :: 581.74 545.31
557.46 20.55 519.50 21.95 :: 578.01 541.45
549.00 13.40 511.09 14.21 :: 562.40 525.30