read/read page cache/read direct/read mmap

fio.cfg

[global]
size=512m
blocksize=32k
timeout=60
numjobs=1
invalidate=1 ; invalidate page-cache (set it explicitly, even though default is true)

[read_mmap]
stonewall
ioengine=mmap
rw=read

[read_regular]
stonewall
rw=read

[read_regular_page_cache]
stonewall
invalidate=0
rw=read

[read_direct]
stonewall
direct=1
rw=read

fio_hdd_512m.log

$ fio fio.cfg |& fgrep -B1 read:
read_mmap: (groupid=0, jobs=1): err= 0: pid=15407: Mon Apr 19 04:03:57 2021
  read: IOPS=10.7k, BW=333MiB/s (350MB/s)(512MiB/1536msec)
--
read_regular: (groupid=1, jobs=1): err= 0: pid=15513: Mon Apr 19 04:03:57 2021
  read: IOPS=10.4k, BW=325MiB/s (341MB/s)(512MiB/1575msec)
--
read_regular_page_cache: (groupid=2, jobs=1): err= 0: pid=15544: Mon Apr 19 04:03:57 2021
  read: IOPS=155k, BW=4830MiB/s (5065MB/s)(512MiB/106msec)
--
read_direct: (groupid=3, jobs=1): err= 0: pid=15556: Mon Apr 19 04:03:57 2021
  read: IOPS=7976, BW=249MiB/s (261MB/s)(512MiB/2054msec)

fio_hdd_5g.log

$ fio fio.cfg |& fgrep -B1 read:
read_mmap: (groupid=0, jobs=1): err= 0: pid=14345: Mon Apr 19 04:03:33 2021
  read: IOPS=10.3k, BW=322MiB/s (338MB/s)(5120MiB/15902msec)
--
read_regular: (groupid=1, jobs=1): err= 0: pid=14355: Mon Apr 19 04:03:33 2021
  read: IOPS=10.6k, BW=330MiB/s (346MB/s)(5120MiB/15507msec)
--
read_regular_page_cache: (groupid=2, jobs=1): err= 0: pid=14428: Mon Apr 19 04:03:33 2021
  read: IOPS=163k, BW=5105MiB/s (5353MB/s)(5120MiB/1003msec)
--
read_direct: (groupid=3, jobs=1): err= 0: pid=14429: Mon Apr 19 04:03:33 2021
  read: IOPS=7900, BW=247MiB/s (259MB/s)(5120MiB/20737msec)

fio_ssd_512m.log

$ fio fio.cfg |& fgrep -B1 read:
read_mmap: (groupid=0, jobs=1): err= 0: pid=18012: Mon Apr 19 11:01:16 2021
  read: IOPS=29.9k, BW=934MiB/s (980MB/s)(512MiB/548msec)
--
read_regular: (groupid=1, jobs=1): err= 0: pid=18013: Mon Apr 19 11:01:16 2021
  read: IOPS=30.5k, BW=952MiB/s (998MB/s)(512MiB/538msec)
--
read_regular_page_cache: (groupid=2, jobs=1): err= 0: pid=18172: Mon Apr 19 11:01:16 2021
  read: IOPS=231k, BW=7211MiB/s (7562MB/s)(512MiB/71msec)
--
read_direct: (groupid=3, jobs=1): err= 0: pid=18173: Mon Apr 19 11:01:16 2021
  read: IOPS=16.7k, BW=522MiB/s (548MB/s)(512MiB/980msec)

fio_ssd_5g.log

$ fio fio.cfg |& fgrep -B1 read:                                                                
read_mmap: (groupid=0, jobs=1): err= 0: pid=19825: Mon Apr 19 11:02:11 2021                                                              
  read: IOPS=32.4k, BW=1013MiB/s (1062MB/s)(5120MiB/5053msec)                                                                            
--                                                                                                                                       
read_regular: (groupid=1, jobs=1): err= 0: pid=19985: Mon Apr 19 11:02:11 2021                                                           
  read: IOPS=32.2k, BW=1008MiB/s (1057MB/s)(5120MiB/5081msec)                                                                            
--                                                                                                                                       
read_regular_page_cache: (groupid=2, jobs=1): err= 0: pid=20327: Mon Apr 19 11:02:11 2021                                                
  read: IOPS=245k, BW=7653MiB/s (8025MB/s)(5120MiB/669msec)                                                                              
--                                                                                                                                       
read_direct: (groupid=3, jobs=1): err= 0: pid=20328: Mon Apr 19 11:02:11 2021                                                            
  read: IOPS=16.4k, BW=514MiB/s (539MB/s)(5120MiB/9965msec)