Large range test

gistfile1.txt

# Setup
roachprod create -n 7 --gce-machine-type=n2d-standard-16 --local-ssd=true $CLUSTER
roachprod put $CLUSTER ./artifacts/cockroach
roachprod start $CLUSTER:1-6

# Init ranges - do not use --splits - it becomes invasive.
roachprod run $CLUSTER:1 "./cockroach workload init kv --splits 0"

# Write some initial data 
roachprod run $CLUSTER:7 "./cockroach workload run kv --duration=30m --read-percent=50 --max-block-bytes=4096 --concurrency=2048 $(roachprod pgurl $CLUSTER:1-6)"

# Enable tracing
SET CLUSTER SETTING sql.trace.stmt.enable_threshold="1000ms";

# Run test - this is about 20-30% of capacity of system.
# Ramp is important so lease acq not included in results, 2m is short enough to capture slowness
roachprod run $CLUSTER:7 "./cockroach workload run kv --ramp=10s --duration=2m --read-percent=50 --max-rate=20000 --max-block-bytes=4096 --write-seq=R1000000 --concurrency=2048 $(roachprod pgurl $CLUSTER:1-6)"

# view range sizes 
select count(range_size) as count, round(max(range_size/1024/1024)) as max, round(avg(range_size)/1024/1024) as avg, round(min(range_size/1024/1024)) as min from crdb_internal.ranges where start_key > '\xf0';

# Settings Increase range size
ALTER DATABASE kv CONFIGURE ZONE USING
range_min_bytes = pow(2,27),
range_max_bytes = pow(2,29);


# Results
# 256MB range split
I230320 21:24:59.429636 30593183 kv/kvserver/replica_command.go:410 ⋮ [T1,n2,split,s2,r71/3:‹/Table/106/1/-5{57055…-38791…}›] 10770  initiating a split of this range at key /Table/106/1/‹-5479273273185716762› [r237] (‹512 MiB above threshold size 512 MiB›)‹›

I230320 21:25:01.225028 30536288 sql/conn_executor_exec.go:2652 ⋮ [T1,n2,client=10.142.1.30:44088,user=root] 10781 +‹     1.110ms      0.056ms                    event:kv/kvserver/spanlatch/manager.go:533 [n1,s1,r71/1:/Table/106/1/-5
{57055…-38791…}] waiting to acquire write latch /Table/106/1/-5561853824373583014/0@1679347499.524297549,0, held by read latch /Table/106/1/-5{570551428199419088-479273273185716762}@0,0›
I230320 21:25:01.225028 30536288 sql/conn_executor_exec.go:2652 ⋮ [T1,n2,client=10.142.1.30:44088,user=root] 10781 +‹  1668.809ms   1667.699ms                    event:kv/kvserver/concurrency/concurrency_manager.go:319 [n1,s1,r71/1:/
Table/106/1/-5{57055…-47927…}] scanning lock table for conflicting locks›

#merge - very little impact
SET CLUSTER SETTING kv.snapshot_rebalance.max_rate = '256 MiB'
SET CLUSTER SETTING kv.range_merge.queue_interval = '10ms';

# Results
Baseline
200  - 120.0s        0         600018         5000.0      3.0      2.5      6.0      8.1     75.5
400  - 120.0s        0         600017         5000.0      2.9      2.5      5.8      7.9     52.4
1600 - 120.0s        0         600016         5000.0      2.9      2.5      5.8      7.6     52.4

Split
100-200 -> 50-100 = 
_elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result
  120.0s        0         600265         5002.1      3.1      2.6      6.0      8.9    125.8



Merge (with 256MB rebalance rate and no wait)
           _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result
100  ->  200  120.0s        0         600012         5000.1      2.9      2.5      5.8      7.6     71.3
200  ->  400  120.0s        0         600009         5000.1      3.3      2.6      6.6     18.9    142.6
400  ->  800  120.0s        0         600003         5000.0      2.9      2.5      5.8      8.1    121.6
800  -> 1600  120.0s        0         600002         5000.1      2.8      2.4      5.5      7.6     65.0
1600 -> 3200
800 -> 1600