res.aws.arm

AWS.arm

Example command line:

raw:
fio --filename=/dev/nvme1n1 --direct=1 --rw=randread --bs=4096 --ioengine=psync --iodepth=1 --runtime=600 --numjobs=1 --time_based --group_reporting --name=iops-test-job --eta-newline=1 --eta-interval=1 --randseed=1669228852 --readonly --eta=always

dir:
fio --filename=/data/m/fio.in.1:/data/m/fio.in.2:/data/m/fio.in.3:/data/m/fio.in.4:/data/m/fio.in.5:/data/m/fio.in.6:/data/m/fio.in.7:/data/m/fio.in.8 --filesize=307200m --rw=randread   --bs=4096 --ioengine=psync --iodepth=1   --time_based --group_reporting   --name=iops-test-job --eta-newline=1 --eta-interval=1   --randrepeat=0 --norandommap   --invalidate=0   --eta=always --direct=1 --numjobs=1 --runtime=600

buf:
fio --filename=/data/m/fio.in.1:/data/m/fio.in.2:/data/m/fio.in.3:/data/m/fio.in.4:/data/m/fio.in.5:/data/m/fio.in.6:/data/m/fio.in.7:/data/m/fio.in.8 --filesize=307200m --rw=randread   --bs=4096 --ioengine=psync --iodepth=1   --time_based --group_reporting   --name=iops-test-job --eta-newline=1 --eta-interval=1   --randrepeat=0 --norandommap   --invalidate=0   --eta=always --direct=0 --fadvise_hint=random --numjobs=1 --runtime=600

IOPs per fio / QPS per db_bench
1	2	5	10	20	30	40	60	<- njobs
3589	7194	17962	35830	70299	80093	80100	80126	fio.4kb.raw.psync
3598	7175	17927	35512	69817	80092	80096	80095	fio.4kb.dir.psync
3569	7189	18094	36715	72880	84171	84176	84166	fio.4kb.buf.psync
-
3226	6467	16205	32236	63624	80088	80089	80103	fio.8kb.raw.psync
3209	6428	16105	32006	63126	80094	80091	80096	fio.8kb.dir.psync
3246	6509	16548	33256	65612	84052	84040	84044	fio.8kb.buf.psync
-
 986	1973	 4945	 9857	19401	24060	24067	24070	dbb.3000b.cmb0.600mkeys.dir.blob
1341	2741	 7014	14071	27792	34734	34743	34747	dbb.3000b.cmb2048.600mkeys.dir.blob
1414	3204	10218	24347	52112	65889	65892	65908	dbb.3000b.cmb16384.600mkeys.dir.blob
1418	3156	10115	24866	55974	72269	72267	72275	dbb.3000b.cmb24576.600mkeys.dir.blob
1044	2100	 5258	10487	20637	25813	25814	25824	dbb.4096b.cmb0.600mkeys.dir.blob
1298	2660	 6792	13657	26957	33975	33969	33984	dbb.4096b.cmb2048.600mkeys.dir.blob
1421	3161	10025	23881	50993	65910	65913	65923	dbb.4096b.cmb16384.600mkeys.dir.blob
1362	3109	 9934	24201	54471	72225	72235	72241	dbb.4096b.cmb24576.600mkeys.dir.blob
-
2967	5923	14852	29659	58454	77988	77996	78001	dbb.3000b.cmb0.600mkeys.dir.level
 657	1315	 3255	 6372	12132	14041	14055	14052	dbb.3000b.cmb2048.600mkeys.dir.level
2655	5529	14280	28637	56676	78127	78203	78209	dbb.3000b.cmb16384.600mkeys.dir.level
2655	5578	14340	28877	57164	78578	78648	78658	dbb.3000b.cmb24576.600mkeys.dir.level
3111	6224	15592	30898	60776	77703	77710	77712	dbb.4096b.cmb0.600mkeys.dir.level
 547	1092	 2686	 5163	 8612	 8618	 8616	 8615	dbb.4096b.cmb2048.600mkeys.dir.level
2712	5771	14986	30190	59859	77480	77510	77506	dbb.4096b.cmb16384.600mkeys.dir.level
2701	5761	14991	30249	59846	77763	77786	77792	dbb.4096b.cmb24576.600mkeys.dir.level
-
1467	4394	15332	32561	64493	83173	83178	83181	dbb.3000b.cmb0.600mkeys.buf.blob
1553	4441	15092	31857	63171	83091	83097	83102	dbb.3000b.cmb2048.600mkeys.buf.blob
1542	4542	15301	32060	61565	77999	77881	77890	dbb.3000b.cmb16384.600mkeys.buf.blob
1459	4335	14992	32116	62054	77419	77313	77314	dbb.3000b.cmb24576.600mkeys.buf.blob
1485	4453	14868	31089	61333	81715	81721	81720	dbb.4096b.cmb0.600mkeys.buf.blob
1612	4533	14737	30866	61023	81607	81614	81617	dbb.4096b.cmb2048.600mkeys.buf.blob
1447	4319	14749	30628	58842	75487	75343	75366	dbb.4096b.cmb16384.600mkeys.buf.blob
1535	4314	14598	30403	58597	76138	76060	76065	dbb.4096b.cmb24576.600mkeys.buf.blob
-
3093	6182	15246	30183	59667	81550	81549	81551	dbb.3000b.cmb0.600mkeys.buf.level
2407	5203	12869	25553	50368	74012	81235	81233	dbb.3000b.cmb2048.600mkeys.buf.level
2785	6007	14919	129554	58118	81030	81082	81094	dbb.3000b.cmb16384.600mkeys.buf.level
2783	5988	14821	29411	57959	80771	80839	80839	dbb.3000b.cmb24576.600mkeys.buf.level
3160	6308	15610	31003	61284	80009	80018	80017	dbb.4096b.cmb0.600mkeys.buf.level
2102	4868	12119	23823	46994	69197	79348	79355	dbb.4096b.cmb2048.600mkeys.buf.level
2677	6205	15316	30369	59743	79310	79345	79359	dbb.4096b.cmb16384.600mkeys.buf.level
2644	6145	15274	30289	59768	79279	79308	79317	dbb.4096b.cmb24576.600mkeys.buf.level
-
185990	392158	906142	1466701	1646550	1684244	1714230	1658330	dbb.3000b.cmb2048.2mkeys.buf.blob
168529	364432	866513	1550529	1601773	1723067	1584602	1576462	dbb.4096b.cmb2048.2mkeys.buf.blob
-
 91885	301773	736906	1398559	2631863	3723485	4683576	6365114	dbb.3000b.cmb2048.2mkeys.buf.level
 50774	283703	709597	1325178	2542118	3574099	4498411	6140358	dbb.4096b.cmb2048.2mkeys.buf.level

response time per fio/db_bench - usecs
1	2	5	10	20	30	40	60	<- njobs
278.2	277.5	277.9	278.6	284.0	374.1	499.0	748.7	fio.4kb.raw.psync
277.6	278.4	278.6	281.3	286.2	374.3	499.2	748.9	fio.4kb.dir.psync
279.9	277.9	276.0	272.0	274.1	356.1	474.9	712.6	fio.4kb.buf.psync
-
309.6	308.8	308.1	309.8	313.9	374.2	499.0	748.7	fio.8kb.raw.psync
311.4	310.8	310.1	312.1	316.5	374.3	499.2	748.9	fio.8kb.dir.psync
307.7	307.0	301.8	300.4	304.5	356.6	475.7	713.7	fio.8kb.buf.psync
-
1014.1	1012.7	1010.4	1013.7	1029.6	1245.8	1660.7	2490.7	dbb.3000b.cmb0.600mkeys.dir.blob
745.2	729.5	712.4	710.4	719.3	863.2	1150.7	1725.5	dbb.3000b.cmb2048.600mkeys.dir.blob
707.1	624.0	489.1	410.6	383.7	455.2	606.9	910.1	dbb.3000b.cmb16384.600mkeys.dir.blob
704.8	633.5	494.2	402.1	357.2	415.0	553.3	829.9	dbb.3000b.cmb24576.600mkeys.dir.blob
957.5	951.8	950.3	952.7	968.3	1161.4	1547.9	2321.3	dbb.4096b.cmb0.600mkeys.dir.blob
770.2	751.6	735.7	732.0	741.5	882.5	1176.7	1764.3	dbb.4096b.cmb2048.600mkeys.dir.blob
703.3	632.7	498.6	418.6	392.1	455.0	606.7	909.8	dbb.4096b.cmb16384.600mkeys.dir.blob
733.7	643.0	503.2	413.1	367.0	415.3	553.6	830.3	dbb.4096b.cmb24576.600mkeys.dir.blob
-
337.0	337.6	336.6	337.1	342.0	384.6	512.7	769.0	dbb.3000b.cmb0.600mkeys.dir.level
1520.9	1519.1	1534.6	1567.4	1646.4	2133.7	2842.2	4262.2	dbb.3000b.cmb2048.600mkeys.dir.level
376.6	361.7	350.1	349.1	352.8	383.9	511.4	766.9	dbb.3000b.cmb16384.600mkeys.dir.level
376.6	358.4	348.6	346.2	349.8	381.7	508.4	762.6	dbb.3000b.cmb24576.600mkeys.dir.level
321.4	321.3	320.6	323.6	329.0	386.0	514.6	771.8	dbb.4096b.cmb0.600mkeys.dir.level
1827.2	1830.1	1860.3	1934.7	2318.7	3475.3	4633.1	6947.4	dbb.4096b.cmb2048.600mkeys.dir.level
368.7	346.5	333.6	331.1	334.0	387.1	515.9	773.9	dbb.4096b.cmb16384.600mkeys.dir.level
370.2	347.1	333.5	330.5	334.1	386.7	514.1	771.1	dbb.4096b.cmb24576.600mkeys.dir.level
-
681.4	455.1	326.1	307.0	310.1	360.6	480.8	721.1	dbb.3000b.cmb0.600mkeys.buf.blob
643.6	450.3	331.2	313.8	316.5	360.9	481.2	721.8	dbb.3000b.cmb2048.600mkeys.buf.blob
648.3	440.3	326.7	311.8	324.8	384.5	513.5	770.1	dbb.3000b.cmb16384.600mkeys.buf.blob
685.2	461.2	333.4	311.3	322.2	387.4	517.2	775.8	dbb.3000b.cmb24576.600mkeys.buf.blob
673.2	449.1	336.2	321.6	326.0	367.0	489.3	734.0	dbb.4096b.cmb0.600mkeys.buf.blob
620.2	441.2	339.3	323.9	327.6	367.5	490.0	734.9	dbb.4096b.cmb2048.600mkeys.buf.blob
690.7	463.0	338.9	326.4	339.8	397.3	530.8	795.9	dbb.4096b.cmb16384.600mkeys.buf.blob
651.3	463.6	342.4	328.8	341.2	393.9	525.8	788.5	dbb.4096b.cmb24576.600mkeys.buf.blob
-
323.2	323.5	327.9	331.2	335.1	367.8	490.3	735.5	dbb.3000b.cmb0.600mkeys.buf.level
415.4	384.4	388.5	391.2	397.0	405.2	492.2	738.4	dbb.3000b.cmb2048.600mkeys.buf.level
359.0	332.9	335.1	338.2	344.0	370.1	493.2	739.7	dbb.3000b.cmb16384.600mkeys.buf.level
359.2	333.9	337.3	339.9	345.0	371.3	494.7	742.0	dbb.3000b.cmb24576.600mkeys.buf.level
316.4	317.0	320.3	322.5	326.2	374.9	499.8	749.6	dbb.4096b.cmb0.600mkeys.buf.level
475.5	410.8	412.5	419.7	425.4	433.4	503.9	755.8	dbb.4096b.cmb2048.600mkeys.buf.level
373.5	322.2	326.4	329.2	334.7	378.2	504.0	755.8	dbb.4096b.cmb16384.600mkeys.buf.level
378.1	325.4	327.3	330.1	334.5	378.3	504.2	756.2	dbb.4096b.cmb24576.600mkeys.buf.level
-
  5.4	  5.1	  5.5	  6.8	 12.1	 17.8	 23.3	 36.2	dbb.3000b.cmb2048.2mkeys.buf.blob
  5.9	  5.5	  5.8	  6.4	 12.5	 17.4	 25.2	 38.1	dbb.4096b.cmb2048.2mkeys.buf.blob
-
 10.9	  6.6	  6.8	  7.2	  7.6	  8.1	  8.5	  9.4	dbb.3000b.cmb2048.2mkeys.buf.level
 19.7	  7.1	  7.0	  7.5	  7.9	  8.4	  8.9	  9.8	dbb.4096b.cmb2048.2mkeys.buf.level

vmstat us+sy
1	2	5	10	20	30	40	60	<- njobs
0	0	0	0	0	0	0.1	0.1	fio.4kb.raw.psync
0	0	0	0	0	0	0	0	fio.4kb.dir.psync
0	0	0	0	1.0	1.0	1.0	1.0	fio.4kb.buf.psync
-
0	0	0	0	0	0	0	0	fio.4kb.raw.psync
0	0	0	0	0	0	0	0	fio.8kb.dir.psync
0	0	0	0.4	1.0	1.0	1.0	1.0	fio.8kb.buf.psync
-
0.0	0.0	0.0	0.0	0.0	0.4	0.4	0.4	dbb.3000b.cmb0.600mkeys.dir.blob
0.0	0.0	0.0	0.0	1.0	1.0	1.0	1.0	dbb.3000b.cmb2048.600mkeys.dir.blob
0.0	0.0	0.0	0.5	1.0	1.2	1.2	1.2	dbb.3000b.cmb16384.600mkeys.dir.blob
0.0	0.0	0.0	0.6	1.0	1.6	1.7	1.7	dbb.3000b.cmb24576.600mkeys.dir.blob
0.0	0.0	0.0	0.0	0.1	0.6	0.6	0.6	dbb.4096b.cmb0.600mkeys.dir.blob
0.0	0.0	0.0	0.0	1.0	1.1	1.0	1.1	dbb.4096b.cmb2048.600mkeys.dir.blob
0.0	0.0	0.0	0.3	1.0	1.2	1.2	1.3	dbb.4096b.cmb16384.600mkeys.dir.blob
0.0	0.0	0.0	0.6	1.0	2.4	2.3	2.4	dbb.4096b.cmb24576.600mkeys.dir.blob
-
0.0	0.0	0.0	0.1	1.0	1.1	1.1	1.1	dbb.3000b.cmb0.600mkeys.dir.level	
0.0	0.0	0.0	0.0	1.0	1.2	1.2	1.3	dbb.3000b.cmb2048.600mkeys.dir.level
0.0	0.0	0.0	1.0	1.0	2.4	2.4	2.6	dbb.3000b.cmb16384.600mkeys.dir.level
0.0	0.0	0.0	1.0	1.1	2.3	2.5	2.7	dbb.3000b.cmb24576.600mkeys.dir.level
0.0	0.0	0.0	0.1	1.0	1.0	1.1	1.1	dbb.4096b.cmb0.600mkeys.dir.level
0.0	0.0	0.0	0.1	1.0	1.0	1.0	1.0	dbb.4096b.cmb2048.600mkeys.dir.level
0.0	0.0	0.0	1.0	1.0	2.3	2.4	2.4	dbb.4096b.cmb16384.600mkeys.dir.level
0.0	0.0	0.0	1.0	1.1	2.3	2.4	2.4	dbb.4096b.cmb24576.600mkeys.dir.level
-
0.0	0.0	0.0	1.0	2.0	3.0	3.0	3.0	dbb.3000b.cmb0.600mkeys.buf.blob
0.0	0.0	0.0	2.0	2.0	4.0	4.0	4.0	dbb.3000b.cmb2048.600mkeys.buf.blob
0.0	0.0	0.0	1.6	2.0	2.5	2.5	2.6	dbb.3000b.cmb16384.600mkeys.buf.blob
0.0	0.0	0.1	1.5	2.0	2.3	2.3	2.4	dbb.3000b.cmb24576.600mkeys.buf.blob
0.0	0.0	0.0	1.2	2.6	3.0	3.0	3.0	dbb.4096b.cmb0.600mkeys.buf.blob
0.0	0.0	0.0	2.0	2.1	4.0	4.0	4.0	dbb.4096b.cmb2048.600mkeys.buf.blob
0.0	0.0	0.0	1.6	2.0	2.5	2.6	2.7	dbb.4096b.cmb16384.600mkeys.buf.blob
0.0	0.0	0.1	1.6	2.0	2.9	2.9	3.0	dbb.4096b.cmb24576.600mkeys.buf.blob
-
0.0	0.0	0.0	0.7	2.0	2.2	2.2	2.3	dbb.3000b.cmb0.600mkeys.buf.level
0.0	0.1	1.8	3.0	6.0	9.2	10.5	10.6	dbb.3000b.cmb2048.600mkeys.buf.level
0.0	0.0	0.0	1.7	2.0	3.3	3.4	3.4	dbb.3000b.cmb16384.600mkeys.buf.level
0.0	0.0	0.0	1.7	2.0	3.4	3.5	3.7	dbb.3000b.cmb24576.600mkeys.buf.level
0.1	0.1	0.1	0.4	2.0	2.0	2.0	2.0	dbb.4096b.cmb0.600mkeys.buf.level
0.0	0.9	2.1	4.2	9.0	13.1	15.2	15.3	dbb.4096b.cmb2048.600mkeys.buf.level
0.0	0.0	0.1	1.6	2.0	3.0	3.1	3.1	dbb.4096b.cmb16384.600mkeys.buf.level
0.0	0.0	0.1	1.5	2.0	3.0	3.0	3.0	dbb.4096b.cmb24576.600mkeys.buf.level
-
1.6	3.0	7.9	15.4	29.1	44.9	60.8	92.2	dbb.3000b.cmb2048.2mkeys.buf.blob
1.6	3.0	8.0	15.8	29.2	45.2	60.8	92.2	dbb.4096b.cmb2048.2mkeys.buf.blob
-
0.6	3.0	7.8	15.4	30.9	46.0	61.1	91.2	dbb.3000b.cmb2048.2mkeys.buf.level
0.4	3.0	7.5	15.8	30.8	46.1	61.1	91.2	dbb.4096b.cmb2048.2mkeys.buf.level

cpu_usecs per operation
1	2	5	10	20	30	40	60	<- njobs
3.4	3.6	3.4	3.5	3.9	4.0	4.2	5.0	fio.4kb.raw.psync
3.8	4.0	4.0	4.0	4.1`	4.5	4.3	4.8	fio.4kb.dir.psync
5.7	6.0	6.1	6.3	6.7	7.0	7.0	7.4	fio.4kb.buf.psync
-
3.6	3.8	3.8	3.8	3.8	4.0	4.1	4.5	fio.8kb.raw.psync
4.2	4.4	4.4	4.4	4.4	4.8	4.6	5.2	fio.8kb.dir.psync
7.9	8.3	8.6	8.9	9.0	9.2	9.1	9.5	fio.8kb.buf.psync
-
23.3	24.2	24.2	24.4	24.3	24.5	24.7	25.0	dbb.3000b.cmb0.600mkeys.dir.blob
22.6	22.8	22.5	22.3	22.4	22.5	22.7	22.8	dbb.3000b.cmb2048.600mkeys.dir.blob
24.6	23.3	20.5	18.5	17.7	17.6	17.7	17.9	dbb.3000b.cmb16384.600mkeys.dir.blob
24.8	24.5	21.1	19.0	17.6	17.4	17.5	17.6	dbb.3000b.cmb24576.600mkeys.dir.blob
23.0	23.2	23.3	23.5	23.5	23.5	23.7	24.0	dbb.4096b.cmb0.600mkeys.dir.blob
23.7	24.0	23.7	23.7	23.6	23.7	23.8	24.0	dbb.4096b.cmb2048.600mkeys.dir.blob
23.0	22.5	19.6	17.6	16.8	16.6	16.6	16.8	dbb.4096b.cmb16384.600mkeys.dir.blob
26.2	24.7	`21.8	19.9	18.5	18.2	18.3	18.4	dbb.4096b.cmb24576.600mkeys.dir.blob
-
15.5	14.5	13.4	12.9	12.7	12.6	12.6	12.6	dbb.3000b.cmb0.600mkeys.dir.level
60.6	59.2	57.7	58.2	58.0	58.1	58.2	58.7	dbb.3000b.cmb2048.600mkeys.dir.level
21.7	20.3	18.8	18.9	18.5	18.5	18.6	17.7	dbb.3000b.cmb16384.600mkeys.dir.level
21.7	21.1	19.4	19.1	18.8	18.6	18.8	18.8	dbb.3000b.cmb24576.600mkeys.dir.level
17.4	15.5	13.8	13.2	13.0	13.0	13.0	13.0	dbb.4096b.cmb0.600mkeys.dir.level
83.2	82.2	79.9	79.8	80.0	81.4	82.1	83.7	dbb.4096b.cmb2048.600mkeys.dir.level
23.0	20.8	19.3	19.0	18.5	18.5	18.6	18.7	dbb.4096b.cmb16384.600mkeys.dir.level
23.4	21.5	19.5	19.0	18.7	18.6	18.6	18.7	dbb.4096b.cmb24576.600mkeys.dir.level
-
24.2	22.2	19.8	19.7	19.6	19.7	19.7	19.9	dbb.3000b.cmb0.600mkeys.buf.blob
30.0	27.6	25.6	25.5	25.2	25.5	25.5	25.8	dbb.3000b.cmb2048.600mkeys.buf.blob
32.7	27.5	22.9	21.7	21.5	21.7	21.6	22.0	dbb.3000b.cmb16384.600mkeys.buf.blob
31.3	27.4	22.5	21.0	20.5	20.6	20.7	20.9	dbb.3000b.cmb24576.600mkeys.buf.blob
28.0	24.8	23.5	23.2	23.3	23.3	23.5	23.6	dbb.4096b.cmb0.600mkeys.buf.blob
30.1	28.0	26.2	26.1	26.0	26.2	26.3	26.5	dbb.4096b.cmb2048.600mkeys.buf.blob
31.8	27.8	23.3	22.3	22.1	22.2	22.4	22.7	dbb.4096b.cmb16384.600mkeys.buf.blob
32.9	30.0	24.9	23.8	23.3	23.4	23.8	23.9	dbb.4096b.cmb24576.600mkeys.buf.blob
-
20.3	18.9	17.8	17.7	17.5	17.6	17.6	17.8	dbb.3000b.cmb0.600mkeys.buf.level
77.4	77.2	77.4	77.6	78.8	80.6	80.9	81.5	dbb.3000b.cmb2048.600mkeys.buf.level
27.3	24.7	23.2	22.9	22.9	22.9	23.0	23.3	dbb.3000b.cmb16384.600mkeys.buf.level
27.6	25.7	23.9	23.5	23.2	23.2	23.3	23.5	dbb.3000b.cmb24576.600mkeys.buf.level
22.7	19.7	17.7	17.2	16.8	17.1	17.1	17.2	dbb.4096b.cmb0.600mkeys.buf.level
113.2	112.4	112.8	114.8	117.4	120.8	121.8	122.4	dbb.4096b.cmb2048.600mkeys.buf.level
28.8	24.8	23.1	23.0	22.9	22.9	23.1	23.4	dbb.4096b.cmb16384.600mkeys.buf.level
29.4	25.2	23.3	22.9	22.5	22.6	22.7	22.9	dbb.4096b.cmb24576.600mkeys.buf.level
-
4.6	5.1	5.5	6.7	11.4	17.0	22.6	35.3	dbb.3000b.cmb2048.2mkeys.buf.blob
4.9	5.5	5.8	6.4	11.8	16.8	24.5	37.2	dbb.4096b.cmb2048.2mkeys.buf.blob
-
6.5	6.6	6.8	7.1	 7.5	 7.9	 8.3	 9.1	dbb.3000b.cmb2048.2mkeys.buf.level
7.0	7.0	7.0	7.5	 7.8	 8.2	 8.7	 9.4	dbb.4096b.cmb2048.2mkeys.buf.level

iostat r_await in usecs
1	2	5	10	20	30	40	60	<- njobs
278	275	276	279	280	370	500	750	fio.4kb.raw.psync
275	277	277	280	280	370	500	749	fio.4kb.dir.psync
278	275	279	280	281	370	490	740	fio.4kb.buf.psync
-
310	308	307	310	310	370	500	750	fio.8kb.raw.psync
310	310	310	310	310	370	500	748	fio.8kb.dir.psync
301	303	306	310	310	370	490	740	fio.8kb.buf.psync
-
300	300	300	300	300	370	490	740	dbb.3000b.cmb0.600mkeys.dir.blob
300	300	300	300	300	370	490	740	dbb.3000b.cmb2048.600mkeys.dir.blob
297	293	291	292	300	359	489	738	dbb.3000b.cmb16384.600mkeys.dir.blob
300	300	296	295	300	359	482	730	dbb.3000b.cmb24576.600mkeys.dir.blob
301	300	300	300	308	370	490	740	dbb.4096b.cmb0.600mkeys.dir.blob
300	300	300	300	306	370	490	740	dbb.4096b.cmb2048.600mkeys.dir.blob
300	300	300	300	303	360	489	738	dbb.4096b.cmb16384.600mkeys.dir.blob
301	301	300	300	308	359	481	730	dbb.4096b.cmb24576.600mkeys.dir.blob
-
325	325	324	325	325	364	493	742	dbb.3000b.cmb0.600mkeys.dir.level
601	601	610	621	656	857	1147	1730	dbb.3000b.cmb2048.600mkeys.dir.level
346	333	326	323	331	361	480	730	dbb.3000b.cmb16384.600mkeys.dir.level
346	332	325	322	327	360	479	729	dbb.3000b.cmb24576.600mkeys.dir.level
311	310	310	310	319	368	497	745	dbb.4096b.cmb0.600mkeys.dir.level
686	689	700	727	861	1287	1716	2562	dbb.4096b.cmb2048.600mkeys.dir.level
348	325	310	306	311	364	485	735	dbb.4096b.cmb16384.600mkeys.dir.level
348	324	310	305	313	363	484	735	dbb.4096b.cmb24576.600mkeys.dir.level
-
301	296	299	300	300	360	480	730	dbb.3000b.cmb0.600mkeys.buf.blob
300	295	300	300	300	350	470	720	dbb.3000b.cmb2048.600mkeys.buf.blob
300	293	297	300	300	352	481	730	dbb.3000b.cmb16384.600mkeys.buf.blob
301	295	298	301	299	358	479	727	dbb.3000b.cmb24576.600mkeys.buf.blob
301	300	302	306	310	360	480	730	dbb.4096b.cmb0.600mkeys.buf.blob
300	300	301	302	310	350	470	720	dbb.4096b.cmb2048.600mkeys.buf.blob
301	300	300	302	309	351	479	730	dbb.4096b.cmb16384.600mkeys.buf.blob
303	300	304	309	309	350	478	729	dbb.4096b.cmb24576.600mkeys.buf.blob
-
323	320	320	321	325	363	483	731	dbb.3000b.cmb0.600mkeys.buf.level
347	320	320	320	324	330	420	671	dbb.3000b.cmb2048.600mkeys.buf.level
343	320	320	320	330	351	480	729	dbb.3000b.cmb16384.600mkeys.buf.level
344	319	319	320	328	350	479	727	dbb.3000b.cmb24576.600mkeys.buf.level
315	308	304	309	311	364	483	730	dbb.4096b.cmb0.600mkeys.buf.level
355	309	305	304	310	311	383	633	dbb.4096b.cmb2048.600mkeys.buf.level
354	307	302	304	310	352	482	732	dbb.4096b.cmb16384.600mkeys.buf.level
353	305	302	306	310	351	481	733	dbb.4096b.cmb24576.600mkeys.buf.level

iostat reads per query
1	2	5	10	20	30	40	60	<- njobs
~1	~1	~1	~1	~1	~1	~1	~1	fio.4kb.raw.psync
~1	~1	~1	~1	~1	~1	~1	~1	fio.4kb.dir.psync
1.0	0.99	0.98	0.96	0.95	0.95	0.95	0.95	fio.4kb.buf.psync
-
~1	~1	~1	~1	~1	~1	~1	~1	fio.8kb.raw.psync
~1	~1	~1	~1	~1	~1	~1	~1	fio.8kb.dir.psync
0.99	0.99	0.97	0.95	0.95	0.95	0.95	0.95	fio.8kb.buf.psync
-
3.3	3.3	3.3	3.3	3.3	3.3	3.3	3.3	dbb.3000b.cmb0.600mkeys.dir.blob
2.4	2.4	2.3	2.3	2.3	2.3	2.3	2.3	dbb.3000b.cmb2048.600mkeys.dir.blob
2.3	2.0	1.6	1.3	1.2	1.2	1.2	1.2	dbb.3000b.cmb16384.600mkeys.dir.blob
2.3	2.1	1.6	1.3	1.1	1.1	1.1	1.1	dbb.3000b.cmb24576.600mkeys.dir.blob
3.1	3.1	3.1	3.1	3.1	3.1	3.1	3.1	dbb.4096b.cmb0.600mkeys.dir.blob
2.5	2.4	2.4	2.4	2.4	2.4	2.4	2.4	dbb.4096b.cmb2048.600mkeys.dir.blob
2.3	2.1	1.6	1.3	1.2	1.2	1.2	1.2	dbb.4096b.cmb16384.600mkeys.dir.blob
2.4	2.1	1.6	1.3	1.1	1.1	1.1	1.1	dbb.4096b.cmb24576.600mkeys.dir.blob
-
1.0	1.0	1.0	1.0	1.0	1.0	1.0	1.0	dbb.3000b.cmb0.600mkeys.dir.level
2.5	2.5	2.4	2.4	2.4	2.4	2.4	2.4	dbb.3000b.cmb2048.600mkeys.dir.level
1.1	1.0	1.0	1.0	1.0	1.0	1.0	1.0	dbb.3000b.cmb16384.600mkeys.dir.level
1.1	1.0	1.0	1.0	1.0	1.0	1.0	1.0	dbb.3000b.cmb24576.600mkeys.dir.level
1.1	1.0	1.0	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb0.600mkeys.dir.level
3.8	3.7	3.7	3.6	3.6	3.6	3.6	3.6	dbb.4096b.cmb2048.600mkeys.dir.level
1.1	1.1	1.0	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb16384.600mkeys.dir.level
1.1	1.1	1.0	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb24576.600mkeys.dir.level
-
2.2	1.5	1.0	0.97	0.96	0.96	0.96	0.96	dbb.3000b.cmb0.600mkeys.buf.blob
2.1	1.4	1.0	0.97	0.96	0.96	0.96	0.96	dbb.3000b.cmb2048.600mkeys.buf.blob
2.1	1.4	1.0	0.98	1.0	1.0	1.0	1.0	dbb.3000b.cmb16384.600mkeys.buf.blob
2.2	1.5	1.1	0.98	1.0	1.0	1.0	1.0	dbb.3000b.cmb24576.600mkeys.buf.blob
2.2	1.4	1.0	0.98	0.98	0.98	0.98	0.98	dbb.4096b.cmb0.600mkeys.buf.blob
2.0	1.4	1.0	0.98	0.98	0.98	0.98	0.98	dbb.4096b.cmb2048.600mkeys.buf.blob
2.2	1.5	1.0	1.0	1.0	1.1	1.0	1.1	dbb.4096b.cmb16384.600mkeys.buf.blob
2.0	1.4	1.0	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb24576.600mkeys.buf.blob
-
0.98	0.96	0.98	0.99	0.98	0.98	0.98	0.98	dbb.3000b.cmb0.600mkeys.buf.level
1.0	0.96	0.98	0.98	0.99	0.99	0.98	0.99	dbb.3000b.cmb2048.600mkeys.buf.level
0.99	0.97	0.98	0.99	0.99	0.98	0.98	0.98	dbb.3000b.cmb16384.600mkeys.buf.level
0.99	0.97	0.98	0.99	0.99	0.99	0.98	0.98	dbb.3000b.cmb24576.600mkeys.buf.level
1.0	0.98	1.0	1.0	1.0	0.99	0.99	0.99	dbb.4096b.cmb0.600mkeys.buf.level
1.1	0.98	0.99	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb2048.600mkeys.buf.level
1.0	0.98	1.0	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb16384.600mkeys.buf.level
1.1	0.99	1.0	1.0	1.0	1.0	1.0	1.0	dbb.4096b.cmb24576.600mkeys.buf.level

iostat rareq-sz in KB
1	2	5	10	20	30	40	60	<- njobs
4.0	4.0	4.0	4.0	4.0	4.0	4.0	4.0	fio.4kb.raw.psync
4.0	4.0	4.0	4.0	4.0	4.0	4.0	4.0	fio.4kb.dir.psync
4.0	4.0	4.0	4.0	4.0	4.0	4.0	4.0	fio.4kb.buf.psync
-
8.0	8.0	8.0	8.0	8.0	8.0	8.0	8.0	fio.8kb.raw.psync
8.0	8.0	8.0	8.0	8.0	8.0	8.0	8.0	fio.8kb.dir.psync
7.7	7.7	7.9	8.0	8.0	8.0	8.0	8.0	fio.8kb.buf.psync
-
4.2	4.2	4.2	4.2	4.2	4.2	4.2	4.2	dbb.3000b.cmb0.600mkeys.dir.blob
4.1	4.1	4.1	4.0	4.0	4.0	4.0	4.0	dbb.3000b.cmb2048.600mkeys.dir.blob
4.1	4.0	3.9	3.7	3.7	3.6	3.6	3.6	dbb.3000b.cmb16384.600mkeys.dir.blob
4.1	4.0	3.9	3.7	3.6	3.6	3.6	3.6	dbb.3000b.cmb24576.600mkeys.dir.blob
4.5	4.5	4.5	4.5	4.5	4.5	4.5	4.5	dbb.4096b.cmb0.600mkeys.dir.blob
4.5	4.5	4.5	4.5	4.5	4.5	4.5	4.5	dbb.4096b.cmb2048.600mkeys.dir.blob
4.5	4.5	4.5	4.5	4.5	4.5	4.5	4.5	dbb.4096b.cmb16384.600mkeys.dir.blob
4.5	4.5	4.5	4.5	4.5	4.5	4.5	4.5	dbb.4096b.cmb24576.600mkeys.dir.blob
-
7.6	7.6	7.5	7.5	7.5	7.5	7.5	7.5	dbb.3000b.cmb0.600mkeys.dir.level
72.2	72.1	72.1	72.2	72.1	72.2	72.1	72.1	dbb.3000b.cmb2048.600mkeys.dir.level
12.3	9.4	7.6	7.0	6.7	6.6	6.6	6.6	dbb.3000b.cmb16384.600mkeys.dir.level
12.3	9.3	7.6	7.0	6.7	6.6	6.6	6.6	dbb.3000b.cmb24576.600mkeys.dir.level
6.1	6.0	5.9	5.9	5.9	5.9	5.9	5.9	dbb.4096b.cmb0.600mkeys.dir.level
78.6	78.7	78.7	78.7	78.7	78.7	78.7	78.7	dbb.4096b.cmb2048.600mkeys.dir.level
13.9	9.2	6.5	5.6	5.1	5.0	5.0	5.0	dbb.4096b.cmb16384.600mkeys.dir.level
13.8	9.2	6.5	5.5	5.1	5.0	5.0	5.0	dbb.4096b.cmb24576.600mkeys.dir.level
-
7.0	6.5	7.8	6.8	6.8	6.8	6.8	6.8	dbb.3000b.cmb0.600mkeys.buf.blob
7.1	6.5	6.8	6.8	6.8	6.8	6.8	6.8	dbb.3000b.cmb2048.600mkeys.buf.blob
7.0	6.6	6.7	6.8	6.7	6.7	6.7	6.7	dbb.3000b.cmb16384.600mkeys.buf.blob
7.1	6.6	6.7	6.8	6.7	6.7	6.8	6.8	dbb.3000b.cmb24576.600mkeys.buf.blob
7.5	7.3	7.7	7.9	7.9	7.9	7.9	7.9	dbb.4096b.cmb0.600mkeys.buf.blob
7.5	7.3	7.7	7.9	7.9	7.9	7.9	7.9	dbb.4096b.cmb2048.600mkeys.buf.blob
7.5	7.3	7.7	7.8	7.7	7.7	7.7	7.7	dbb.4096b.cmb16384.600mkeys.buf.blob
7.6	7.3	7.7	7.8	7.7	7.7	7.7	7.7	dbb.4096b.cmb24576.600mkeys.buf.blob
-
10.5	9.8	9.7	9.6	9.6	10.5	10.5	10.5	dbb.3000b.cmb0.600mkeys.buf.level
15.3	9.8	9.6	9.6	9.6	9.6	9.6	9.6	dbb.3000b.cmb2048.600mkeys.buf.level
15.2	9.8	9.7	9.6	9.7	9.8	9.8	9.8	dbb.3000b.cmb16384.600mkeys.buf.level
15.3	9.8	9.6	9.7	9.7	9.8	9.8	9.8	dbb.3000b.cmb24576.600mkeys.buf.level
9.0	8.0	7.9	7.9	8.1	8.8	8.8	8.8	dbb.4096b.cmb0.600mkeys.buf.level
19.0	8.0	7.9	7.8	7.8	7.8	7.8	7.8	dbb.4096b.cmb2048.600mkeys.buf.level
18.7	8.0	7.9	7.8	7.9	8.3	8.3	8.3	dbb.4096b.cmb16384.600mkeys.buf.level
18.7	8.0	7.9	7.9	8.0	8.3	8.3	8.3	dbb.4096b.cmb24576.600mkeys.buf.level