HST vs. ST benchmarking

Table of Contents

1 3 paths with equal cost

1.1 Setup

3 iSCSI paths to the same disk, exposed through 3 tgtd instances. All paths have the same cost.

1.2 read

1.2.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42676 	0 	341328 	59704 	0 	0 	0 	0 	
sdb 	42677 	0 	341336 	60574 	0 	0 	0 	0 	
sdc 	42677 	0 	341336 	59691 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42671 	0 	341368 	59298 	0 	0 	0 	0 	
sdb 	42661 	0 	341288 	59334 	0 	0 	0 	0 	
sdc 	42668 	0 	341344 	59343 	0 	0 	0 	0

1.2.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42636 	0 	341048 	56663 	0 	0 	0 	0 	
sdb 	42688 	0 	341464 	56600 	0 	0 	0 	0 	
sdc 	42691 	0 	341488 	56795 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42660 	0 	341280 	56411 	0 	0 	0 	0 	
sdb 	42681 	0 	341448 	56476 	0 	0 	0 	0 	
sdc 	42659 	0 	341272 	56465 	0 	0 	0 	0

1.2.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42617 	0 	340912 	445055 	0 	0 	0 	0 	
sdb 	42706 	0 	341624 	444824 	0 	0 	0 	0 	
sdc 	42686 	0 	341464 	444325 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42645 	0 	341160 	435767 	0 	0 	0 	0 	
sdb 	42615 	0 	340920 	436118 	0 	0 	0 	0 	
sdc 	42740 	0 	341920 	434674 	0 	0 	0 	0

1.2.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42735 	0 	341856 	872222 	0 	0 	0 	0 	
sdb 	42723 	0 	341760 	872748 	0 	0 	0 	0 	
sdc 	42551 	0 	340384 	873036 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42717 	0 	341736 	873039 	0 	0 	0 	0 	
sdb 	42712 	0 	341696 	872991 	0 	0 	0 	0 	
sdc 	42571 	0 	340568 	873777 	0 	0 	0 	0

1.3 write

1.3.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	43 	0 	1136 	72 	42667 	0 	341336 	0 	
sdb 	44 	0 	1616 	68 	42666 	0 	341328 	0 	
sdc 	43 	0 	1504 	78 	42667 	0 	341336 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42711 	0 	341688 	60302 	
sdb 	0 	0 	0 	0 	42669 	0 	341352 	59671 	
sdc 	0 	0 	0 	0 	42620 	0 	340960 	60371

1.3.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	39 	0 	1248 	65 	42681 	0 	341448 	120591 	
sdb 	39 	0 	1984 	60 	42641 	0 	341128 	119888 	
sdc 	37 	0 	1024 	85 	42678 	0 	341424 	120554
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	41975 	0 	335800 	176562 	
sdb 	0 	0 	0 	0 	42871 	0 	342968 	175981 	
sdc 	0 	0 	0 	0 	43154 	0 	345232 	176555

1.3.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	37 	0 	1616 	88 	42605 	0 	340840 	233138 	
sdb 	36 	0 	1504 	90 	42746 	0 	341968 	233649 	
sdc 	36 	0 	1136 	101 	42649 	0 	341192 	234518
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42751 	0 	342008 	691507 	
sdb 	0 	0 	0 	0 	42604 	0 	340832 	692594 	
sdc 	0 	0 	0 	0 	42645 	0 	341160 	693550

1.3.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	36 	0 	1136 	125 	42678 	0 	341424 	1139622 	
sdb 	37 	0 	1984 	118 	42590 	0 	340720 	1141048 	
sdc 	36 	0 	1136 	121 	42732 	0 	341856 	1142448
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42604 	0 	340832 	2042046 	
sdb 	0 	0 	0 	0 	42695 	0 	341560 	2043080 	
sdc 	0 	0 	0 	0 	42701 	0 	341608 	2044316

1.4 randread

1.4.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42676 	0 	341328 	59032 	0 	0 	0 	2939123 	
sdb 	42677 	0 	341336 	59204 	0 	0 	0 	2940710 	
sdc 	42677 	0 	341336 	59011 	0 	0 	0 	2942042
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42674 	0 	341392 	59616 	0 	0 	0 	2939123 	
sdb 	42656 	0 	341248 	59437 	0 	0 	0 	2940710 	
sdc 	42670 	0 	341360 	59548 	0 	0 	0 	2942042

1.4.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42682 	0 	341416 	55416 	0 	0 	0 	2939123 	
sdb 	42657 	0 	341216 	55332 	0 	0 	0 	2940710 	
sdc 	42676 	0 	341368 	55442 	0 	0 	0 	2942042
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42099 	0 	336792 	54480 	0 	0 	0 	2939123 	
sdb 	44257 	0 	354056 	56901 	0 	0 	0 	2940710 	
sdc 	41644 	0 	333152 	54051 	0 	0 	0 	2942042

1.4.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42711 	0 	341664 	440334 	0 	0 	0 	2939123 	
sdb 	42689 	0 	341488 	440003 	0 	0 	0 	2940710 	
sdc 	42609 	0 	340848 	439513 	0 	0 	0 	2942042
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42617 	0 	340936 	441645 	0 	0 	0 	2939123 	
sdb 	42682 	0 	341456 	440636 	0 	0 	0 	2940710 	
sdc 	42701 	0 	341608 	441209 	0 	0 	0 	2942042

1.4.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42620 	0 	340936 	885604 	0 	0 	0 	2939123 	
sdb 	42757 	0 	342032 	886616 	0 	0 	0 	2940710 	
sdc 	42632 	0 	341032 	886217 	0 	0 	0 	2942042
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	42699 	0 	341592 	889889 	0 	0 	0 	2939123 	
sdb 	42712 	0 	341696 	889809 	0 	0 	0 	2940710 	
sdc 	42589 	0 	340712 	889628 	0 	0 	0 	2942042

1.5 randwrite

1.5.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	43 	0 	1136 	73 	42667 	0 	341336 	2939123 	
sdb 	44 	0 	1616 	67 	42666 	0 	341328 	2940710 	
sdc 	43 	0 	1504 	65 	42667 	0 	341336 	2942042
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42673 	0 	341384 	2999508 	
sdb 	0 	0 	0 	0 	42586 	0 	340688 	3000968 	
sdc 	0 	0 	0 	0 	42741 	0 	341928 	3002335

1.5.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	39 	0 	1616 	63 	42675 	0 	341400 	3059718 	
sdb 	39 	0 	1616 	77 	42676 	0 	341408 	3061238 	
sdc 	37 	0 	1024 	64 	42649 	0 	341192 	3062657
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42552 	0 	340416 	3115499 	
sdb 	0 	0 	0 	0 	42822 	0 	342576 	3116984 	
sdc 	0 	0 	0 	0 	42626 	0 	341008 	3118413

1.5.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	37 	0 	1616 	83 	42613 	0 	340904 	3171189 	
sdb 	36 	0 	1504 	83 	42793 	0 	342344 	3172903 	
sdc 	36 	0 	1136 	90 	42594 	0 	340752 	3174100
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42624 	0 	340992 	3620619 	
sdb 	0 	0 	0 	0 	42682 	0 	341456 	3622005 	
sdc 	0 	0 	0 	0 	42694 	0 	341552 	3622738

1.5.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  `- 2:0:0:1 sdc 8:32 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	35 	0 	1432 	114 	42627 	0 	341016 	4069915 	
sdb 	40 	0 	1752 	144 	42621 	0 	340968 	4071839 	
sdc 	34 	0 	1072 	116 	42752 	0 	342016 	4072000
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  `- 2:0:0:1 sdc 8:32 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	42626 	0 	341008 	4968787 	
sdb 	0 	0 	0 	0 	42698 	0 	341584 	4970613 	
sdc 	0 	0 	0 	0 	42676 	0 	341408 	4970351

2 3 paths with equal cost (sda, sdb, sdc), 1 path 4 times worse (sdd)

2.1 Setup

4 iSCSI paths to the same backend disk, exposed through 4 tgtd instances. The first three paths have an artificial delay of 500ms while the last path has a link delay 4 times worse (2s).

2.2 read

2.2.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	32014 	0 	256000 	47712 	0 	0 	0 	5875258 	
sdb 	32014 	0 	256000 	48177 	0 	0 	0 	5876946 	
sdc 	32014 	0 	256000 	48393 	0 	0 	0 	5877308 	
sdd 	32014 	0 	256000 	112967 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	37860 	0 	302880 	54571 	0 	0 	0 	5875258 	
sdb 	37850 	0 	302800 	54368 	0 	0 	0 	5876946 	
sdc 	37857 	0 	302856 	54709 	0 	0 	0 	5877308 	
sdd 	14433 	0 	115464 	50291 	0 	0 	0 	0

2.2.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	32056 	0 	256664 	43850 	0 	0 	0 	5875258 	
sdb 	32060 	0 	256592 	43824 	0 	0 	0 	5876946 	
sdc 	32054 	0 	256680 	43781 	0 	0 	0 	5877308 	
sdd 	31914 	0 	256152 	108651 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	39750 	0 	318000 	52302 	0 	0 	0 	5875258 	
sdb 	40532 	0 	324256 	53112 	0 	0 	0 	5876946 	
sdc 	39968 	0 	319744 	52484 	0 	0 	0 	5877308 	
sdd 	7750 	0 	62000 	25856 	0 	0 	0 	0

2.2.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38230 	0 	305816 	293112 	0 	0 	0 	5875258 	
sdb 	38283 	0 	306240 	293053 	0 	0 	0 	5876946 	
sdc 	38260 	0 	306056 	293029 	0 	0 	0 	5877308 	
sdd 	13239 	0 	105888 	299267 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38287 	0 	306296 	307200 	0 	0 	0 	5875258 	
sdb 	38129 	0 	305032 	307099 	0 	0 	0 	5876946 	
sdc 	38044 	0 	304352 	307249 	0 	0 	0 	5877308 	
sdd 	13540 	0 	108320 	315669 	0 	0 	0 	0

2.2.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38193 	0 	305520 	614370 	0 	0 	0 	5875258 	
sdb 	38242 	0 	305912 	614407 	0 	0 	0 	5876946 	
sdc 	38136 	0 	305064 	614346 	0 	0 	0 	5877308 	
sdd 	13441 	0 	107504 	622123 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38256 	0 	306048 	593007 	0 	0 	0 	5875258 	
sdb 	38255 	0 	306040 	593049 	0 	0 	0 	5876946 	
sdc 	38202 	0 	305616 	592965 	0 	0 	0 	5877308 	
sdd 	13287 	0 	106296 	598808 	0 	0 	0 	0

2.3 write

2.3.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	40 	0 	1280 	58 	32000 	0 	256000 	5875258 	
sdb 	38 	0 	848 	57 	32000 	0 	256000 	5876946 	
sdc 	38 	0 	840 	59 	32000 	0 	256000 	5877308 	
sdd 	40 	0 	1288 	179 	32000 	0 	256000 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	37910 	0 	303280 	5923614 	
sdb 	0 	0 	0 	0 	37682 	0 	301456 	5925051 	
sdc 	0 	0 	0 	0 	37921 	0 	303368 	5924804 	
sdd 	0 	0 	0 	0 	14487 	0 	115896 	112283

2.3.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	31 	0 	840 	46 	32029 	0 	256232 	5978700 	
sdb 	33 	0 	1288 	72 	32029 	0 	256232 	5979430 	
sdc 	31 	0 	848 	51 	32029 	0 	256232 	5979779 	
sdd 	33 	0 	1280 	134 	31913 	0 	255304 	162622
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	40161 	0 	321288 	6022090 	
sdb 	0 	0 	0 	0 	40170 	0 	321360 	6022842 	
sdc 	0 	0 	0 	0 	39782 	0 	318256 	6023137 	
sdd 	0 	0 	0 	0 	7887 	0 	63096 	269865

2.3.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	27 	0 	840 	59 	38150 	0 	305200 	6075293 	
sdb 	30 	0 	1320 	68 	38131 	0 	305048 	6075944 	
sdc 	27 	0 	848 	61 	38236 	0 	305888 	6075822 	
sdd 	29 	0 	1280 	184 	13483 	0 	107864 	296173
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	38115 	0 	304920 	6381178 	
sdb 	0 	0 	0 	0 	38073 	0 	304584 	6382107 	
sdc 	0 	0 	0 	0 	38112 	0 	304896 	6381903 	
sdd 	0 	0 	0 	0 	13700 	0 	109600 	609890

2.3.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	28 	0 	976 	94 	38011 	0 	304088 	6697526 	
sdb 	28 	0 	1016 	91 	38063 	0 	304504 	6698240 	
sdc 	27 	0 	712 	123 	38135 	0 	305080 	6698453 	
sdd 	29 	0 	1552 	250 	13791 	0 	110328 	936634
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	38095 	0 	304760 	7349273 	
sdb 	0 	0 	0 	0 	38011 	0 	304088 	7349644 	
sdc 	0 	0 	0 	0 	38031 	0 	304248 	7349767 	
sdd 	0 	0 	0 	0 	13863 	0 	110904 	1598366

2.4 randread

2.4.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	32013 	0 	256000 	47757 	0 	0 	0 	7998707 	
sdb 	32013 	0 	256000 	47251 	0 	0 	0 	7999985 	
sdc 	32014 	0 	256000 	47989 	0 	0 	0 	7999848 	
sdd 	32013 	0 	256000 	112264 	0 	0 	0 	2258907
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	37889 	0 	303112 	53900 	0 	0 	0 	7998707 	
sdb 	37914 	0 	303312 	53879 	0 	0 	0 	7999985 	
sdc 	37908 	0 	303264 	53873 	0 	0 	0 	7999848 	
sdd 	14289 	0 	114312 	49599 	0 	0 	0 	2258907

2.4.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	32035 	0 	256224 	44285 	0 	0 	0 	7998707 	
sdb 	32034 	0 	256216 	44375 	0 	0 	0 	7999985 	
sdc 	32035 	0 	256224 	44372 	0 	0 	0 	7999848 	
sdd 	31924 	0 	255336 	109261 	0 	0 	0 	2258907
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	40193 	0 	321544 	52370 	0 	0 	0 	7998707 	
sdb 	39741 	0 	317928 	51876 	0 	0 	0 	7999985 	
sdc 	40346 	0 	322768 	52547 	0 	0 	0 	7999848 	
sdd 	7720 	0 	61760 	25677 	0 	0 	0 	2258907

2.4.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38234 	0 	305848 	290212 	0 	0 	0 	7998707 	
sdb 	38338 	0 	306680 	290152 	0 	0 	0 	7999985 	
sdc 	38258 	0 	306040 	290192 	0 	0 	0 	7999848 	
sdd 	13182 	0 	105432 	295052 	0 	0 	0 	2258907
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38230 	0 	305840 	291222 	0 	0 	0 	7998707 	
sdb 	38297 	0 	306376 	291233 	0 	0 	0 	7999985 	
sdc 	38266 	0 	306128 	291236 	0 	0 	0 	7999848 	
sdd 	13207 	0 	105656 	296193 	0 	0 	0 	2258907

2.4.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38201 	0 	305584 	600242 	0 	0 	0 	7998707 	
sdb 	38308 	0 	306440 	599981 	0 	0 	0 	7999985 	
sdc 	38229 	0 	305808 	600203 	0 	0 	0 	7999848 	
sdd 	13274 	0 	106168 	608489 	0 	0 	0 	2258907
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	38233 	0 	305864 	635598 	0 	0 	0 	7998707 	
sdb 	38209 	0 	305672 	635964 	0 	0 	0 	7999985 	
sdc 	37974 	0 	303792 	635877 	0 	0 	0 	7999848 	
sdd 	13584 	0 	108672 	646816 	0 	0 	0 	2258907

2.5 randwrite

2.5.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	40 	0 	1280 	74 	32000 	0 	256000 	7998707 	
sdb 	38 	0 	848 	56 	32000 	0 	256000 	7999985 	
sdc 	38 	0 	840 	54 	32000 	0 	256000 	7999848 	
sdd 	40 	0 	1288 	161 	32000 	0 	256000 	2258907
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	37684 	0 	301472 	8047015 	
sdb 	0 	0 	0 	0 	38043 	0 	304344 	8048245 	
sdc 	0 	0 	0 	0 	37932 	0 	303456 	8047650 	
sdd 	0 	0 	0 	0 	14341 	0 	114728 	2371899

2.5.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	33 	0 	1280 	63 	32032 	0 	256256 	8102212 	
sdb 	33 	0 	1016 	60 	32033 	0 	256264 	8103891 	
sdc 	31 	0 	848 	45 	32031 	0 	256248 	8103355 	
sdd 	31 	0 	1112 	136 	31904 	0 	255232 	2422556
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	40208 	0 	321664 	8148811 	
sdb 	0 	0 	0 	0 	40372 	0 	322976 	8150773 	
sdc 	0 	0 	0 	0 	39721 	0 	317768 	8150457 	
sdd 	0 	0 	0 	0 	7699 	0 	61592 	2535744

2.5.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	25 	0 	952 	112 	38004 	0 	304032 	8208292 	
sdb 	24 	0 	1592 	174 	37964 	0 	303712 	8210815 	
sdc 	24 	0 	632 	103 	38028 	0 	304224 	8209689 	
sdd 	25 	0 	1192 	213 	14004 	0 	112032 	2564584
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	38112 	0 	304896 	8546679 	
sdb 	0 	0 	0 	0 	38087 	0 	304696 	8549467 	
sdc 	0 	0 	0 	0 	38155 	0 	305240 	8548326 	
sdd 	0 	0 	0 	0 	13646 	0 	109168 	2913116

2.5.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0  active undef running
  |- 1:0:0:1 sdb 8:16 active undef running
  |- 2:0:0:1 sdc 8:32 active undef running
  `- 3:0:0:1 sdd 8:48 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	29 	0 	1280 	89 	38119 	0 	304952 	8861825 	
sdb 	29 	0 	1016 	90 	38006 	0 	304048 	8864400 	
sdc 	27 	0 	848 	86 	38237 	0 	305896 	8863400 	
sdd 	27 	0 	1112 	238 	13638 	0 	109104 	3237477
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0  undef undef running
  |- 1:0:0:1 sdb 8:16 undef undef running
  |- 2:0:0:1 sdc 8:32 undef undef running
  `- 3:0:0:1 sdd 8:48 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	38119 	0 	304952 	9494192 	
sdb 	0 	0 	0 	0 	38269 	0 	306152 	9496502 	
sdc 	0 	0 	0 	0 	38142 	0 	305136 	9495941 	
sdd 	0 	0 	0 	0 	13470 	0 	107760 	3880947

3 10 paths with U cost

3.1 Setup

10 iSCSI paths to the same backend disk, exposed through 10 tgtd instances. The cost is distributed in a U shape, where the cost increases and decreases, as below:

The delay is multiplied by a constant of 500ms and applied to every packet response.

Path Delay multiplier
1 1
2 1
3 1
4 4
5 8
6 8
7 16
8 16
9 1
10 1

3.2 read

3.2.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	12830 	0 	102400 	21099 	0 	0 	0 	10110960 	
sdb 	12830 	0 	102400 	20381 	0 	0 	0 	10113274 	
sdc 	12830 	0 	102400 	19726 	0 	0 	0 	10112792 	
sdd 	12830 	0 	102400 	45157 	0 	0 	0 	4507011 	
sde 	12830 	0 	102400 	78662 	0 	0 	0 	0 	
sdf 	12830 	0 	102400 	79830 	0 	0 	0 	0 	
sdg 	12830 	0 	102400 	149950 	0 	0 	0 	0 	
sdh 	12830 	0 	102400 	150487 	0 	0 	0 	0 	
sdi 	12830 	0 	102400 	22054 	0 	0 	0 	0 	
sdj 	12830 	0 	102400 	21510 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	20572 	0 	164576 	31697 	0 	0 	0 	10110960 	
sdb 	20527 	0 	164216 	31519 	0 	0 	0 	10113274 	
sdc 	20583 	0 	164664 	31696 	0 	0 	0 	10112792 	
sdd 	8979 	0 	71832 	31904 	0 	0 	0 	4507011 	
sde 	5415 	0 	43320 	33224 	0 	0 	0 	0 	
sdf 	5414 	0 	43312 	33233 	0 	0 	0 	0 	
sdg 	2723 	0 	21784 	31410 	0 	0 	0 	0 	
sdh 	2722 	0 	21776 	31384 	0 	0 	0 	0 	
sdi 	20536 	0 	164288 	31643 	0 	0 	0 	0 	
sdj 	20529 	0 	164232 	31643 	0 	0 	0 	0

3.2.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	12815 	0 	102400 	21622 	0 	0 	0 	10110960 	
sdb 	12815 	0 	102400 	21195 	0 	0 	0 	10113274 	
sdc 	12815 	0 	102400 	20688 	0 	0 	0 	10112792 	
sdd 	12815 	0 	102400 	45245 	0 	0 	0 	4507011 	
sde 	12815 	0 	102400 	78268 	0 	0 	0 	0 	
sdf 	12815 	0 	102400 	77842 	0 	0 	0 	0 	
sdg 	12815 	0 	102400 	147128 	0 	0 	0 	0 	
sdh 	12815 	0 	102400 	147312 	0 	0 	0 	0 	
sdi 	12815 	0 	102400 	20866 	0 	0 	0 	0 	
sdj 	12815 	0 	102400 	21618 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	23696 	0 	189568 	31036 	0 	0 	0 	10110960 	
sdb 	23683 	0 	189464 	31034 	0 	0 	0 	10113274 	
sdc 	23651 	0 	189208 	31019 	0 	0 	0 	10112792 	
sdd 	4033 	0 	32264 	13455 	0 	0 	0 	4507011 	
sde 	2011 	0 	16088 	11965 	0 	0 	0 	0 	
sdf 	2011 	0 	16088 	11967 	0 	0 	0 	0 	
sdg 	1012 	0 	8096 	11311 	0 	0 	0 	0 	
sdh 	1011 	0 	8088 	11304 	0 	0 	0 	0 	
sdi 	23427 	0 	187416 	30808 	0 	0 	0 	0 	
sdj 	23465 	0 	187720 	30828 	0 	0 	0 	0

3.2.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21729 	0 	173816 	59125 	0 	0 	0 	10110960 	
sdb 	21724 	0 	173776 	59000 	0 	0 	0 	10113274 	
sdc 	21644 	0 	173136 	59236 	0 	0 	0 	10112792 	
sdd 	7446 	0 	59552 	61723 	0 	0 	0 	4507011 	
sde 	3997 	0 	31960 	68077 	0 	0 	0 	0 	
sdf 	3990 	0 	31904 	69569 	0 	0 	0 	0 	
sdg 	2067 	0 	16520 	79602 	0 	0 	0 	0 	
sdh 	2067 	0 	16520 	79672 	0 	0 	0 	0 	
sdi 	21698 	0 	173568 	59022 	0 	0 	0 	0 	
sdj 	21658 	0 	173248 	59199 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21670 	0 	173360 	60223 	0 	0 	0 	10110960 	
sdb 	21574 	0 	172592 	60254 	0 	0 	0 	10113274 	
sdc 	21627 	0 	173016 	60247 	0 	0 	0 	10112792 	
sdd 	7562 	0 	60496 	62845 	0 	0 	0 	4507011 	
sde 	4053 	0 	32424 	69542 	0 	0 	0 	0 	
sdf 	4052 	0 	32416 	69236 	0 	0 	0 	0 	
sdg 	2100 	0 	16800 	81202 	0 	0 	0 	0 	
sdh 	2103 	0 	16824 	81356 	0 	0 	0 	0 	
sdi 	21634 	0 	173072 	60223 	0 	0 	0 	0 	
sdj 	21625 	0 	173000 	60250 	0 	0 	0 	0

3.2.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=read --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21710 	0 	173672 	124546 	0 	0 	0 	10110960 	
sdb 	21572 	0 	172568 	124630 	0 	0 	0 	10113274 	
sdc 	21608 	0 	172856 	124577 	0 	0 	0 	10112792 	
sdd 	7586 	0 	60680 	136766 	0 	0 	0 	4507011 	
sde 	4064 	0 	32504 	144892 	0 	0 	0 	0 	
sdf 	4044 	0 	32344 	145154 	0 	0 	0 	0 	
sdg 	2109 	0 	16864 	146947 	0 	0 	0 	0 	
sdh 	2108 	0 	16848 	146983 	0 	0 	0 	0 	
sdi 	21613 	0 	172888 	124663 	0 	0 	0 	0 	
sdj 	21598 	0 	172776 	124549 	0 	0 	0 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21568 	0 	172544 	125577 	0 	0 	0 	10110960 	
sdb 	21634 	0 	173072 	125558 	0 	0 	0 	10113274 	
sdc 	21566 	0 	172528 	125584 	0 	0 	0 	10112792 	
sdd 	7631 	0 	61048 	136543 	0 	0 	0 	4507011 	
sde 	4096 	0 	32768 	146216 	0 	0 	0 	0 	
sdf 	4077 	0 	32616 	146329 	0 	0 	0 	0 	
sdg 	2127 	0 	17016 	148185 	0 	0 	0 	0 	
sdh 	2125 	0 	17000 	148112 	0 	0 	0 	0 	
sdi 	21567 	0 	172536 	125503 	0 	0 	0 	0 	
sdj 	21609 	0 	172872 	125517 	0 	0 	0 	0

3.3 write

3.3.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	37 	0 	72 	52 	12800 	0 	102400 	10110960 	
sdb 	39 	0 	104 	56 	12798 	0 	102384 	10113274 	
sdc 	37 	0 	104 	55 	12800 	0 	102400 	10112792 	
sdd 	37 	0 	560 	128 	12800 	0 	102400 	4507011 	
sde 	37 	0 	432 	240 	12800 	0 	102400 	0 	
sdf 	37 	0 	560 	227 	12800 	0 	102400 	0 	
sdg 	35 	0 	40 	405 	12801 	0 	102408 	0 	
sdh 	36 	0 	552 	395 	12800 	0 	102400 	0 	
sdi 	37 	0 	56 	56 	12800 	0 	102400 	0 	
sdj 	36 	0 	48 	47 	12801 	0 	102408 	0
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	20601 	0 	164808 	10132373 	
sdb 	0 	0 	0 	0 	20542 	0 	164336 	10133852 	
sdc 	0 	0 	0 	0 	20644 	0 	165152 	10132480 	
sdd 	0 	0 	0 	0 	8810 	0 	70480 	4551883 	
sde 	0 	0 	0 	0 	5381 	0 	43048 	77830 	
sdf 	0 	0 	0 	0 	5381 	0 	43048 	79079 	
sdg 	0 	0 	0 	0 	2717 	0 	21736 	149129 	
sdh 	0 	0 	0 	0 	2717 	0 	21736 	149911 	
sdi 	0 	0 	0 	0 	20634 	0 	165072 	22067 	
sdj 	0 	0 	0 	0 	20573 	0 	164584 	21818

3.3.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	28 	0 	320 	41 	12800 	0 	102400 	10164110 	
sdb 	27 	0 	560 	45 	12800 	0 	102400 	10165541 	
sdc 	27 	0 	952 	34 	12800 	0 	102400 	10164293 	
sdd 	27 	0 	720 	90 	12800 	0 	102400 	4583278 	
sde 	26 	0 	128 	163 	12801 	0 	102408 	110711 	
sdf 	29 	0 	648 	184 	12798 	0 	102384 	112074 	
sdg 	26 	0 	104 	351 	12801 	0 	102408 	180347 	
sdh 	25 	0 	240 	343 	12801 	0 	102408 	181132 	
sdi 	29 	0 	656 	37 	12799 	0 	102392 	53736 	
sdj 	28 	0 	224 	37 	12800 	0 	102400 	53090
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	23141 	0 	185128 	10185845 	
sdb 	0 	0 	0 	0 	23396 	0 	187168 	10186834 	
sdc 	0 	0 	0 	0 	23624 	0 	188992 	10185065 	
sdd 	0 	0 	0 	0 	4071 	0 	32568 	4628490 	
sde 	0 	0 	0 	0 	2028 	0 	16224 	188447 	
sdf 	0 	0 	0 	0 	2016 	0 	16128 	189466 	
sdg 	0 	0 	0 	0 	1018 	0 	8144 	326616 	
sdh 	0 	0 	0 	0 	1020 	0 	8160 	327650 	
sdi 	0 	0 	0 	0 	24255 	0 	194040 	74401 	
sdj 	0 	0 	0 	0 	23431 	0 	187448 	74573

3.3.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	13 	0 	1744 	25 	21581 	0 	172648 	10216644 	
sdb 	14 	0 	440 	23 	21575 	0 	172600 	10217896 	
sdc 	11 	0 	336 	19 	21617 	0 	172936 	10216386 	
sdd 	9 	0 	240 	38 	7619 	0 	60952 	4642170 	
sde 	9 	0 	88 	73 	4085 	0 	32680 	200542 	
sdf 	8 	0 	168 	61 	4105 	0 	32840 	201537 	
sdg 	7 	0 	288 	116 	2137 	0 	17096 	338014 	
sdh 	8 	0 	56 	144 	2130 	0 	17040 	339061 	
sdi 	8 	0 	552 	13 	21556 	0 	172448 	106206 	
sdj 	12 	0 	672 	22 	21595 	0 	172760 	105739
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	21579 	0 	172632 	10278058 	
sdb 	0 	0 	0 	0 	21589 	0 	172712 	10279301 	
sdc 	0 	0 	0 	0 	21520 	0 	172160 	10277818 	
sdd 	0 	0 	0 	0 	7687 	0 	61496 	4706367 	
sde 	0 	0 	0 	0 	4132 	0 	33056 	270869 	
sdf 	0 	0 	0 	0 	4143 	0 	33144 	272159 	
sdg 	0 	0 	0 	0 	2160 	0 	17280 	420964 	
sdh 	0 	0 	0 	0 	2162 	0 	17296 	422009 	
sdi 	0 	0 	0 	0 	21598 	0 	172784 	167696 	
sdj 	0 	0 	0 	0 	21430 	0 	171440 	167010

3.3.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=write --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	8 	0 	560 	17 	21542 	0 	172336 	10340185 	
sdb 	8 	0 	56 	18 	21525 	0 	172200 	10341292 	
sdc 	9 	0 	568 	22 	21584 	0 	172672 	10339957 	
sdd 	8 	0 	264 	52 	7692 	0 	61536 	4771370 	
sde 	8 	0 	56 	86 	4160 	0 	33280 	343372 	
sdf 	6 	0 	40 	66 	4161 	0 	33288 	344495 	
sdg 	6 	0 	40 	133 	2162 	0 	17296 	504865 	
sdh 	8 	0 	552 	252 	2164 	0 	17312 	505804 	
sdi 	8 	0 	872 	21 	21481 	0 	171848 	229770 	
sdj 	9 	0 	64 	18 	21529 	0 	172232 	229149
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	21474 	0 	171792 	10468388 	
sdb 	0 	0 	0 	0 	21494 	0 	171952 	10469480 	
sdc 	0 	0 	0 	0 	21495 	0 	171960 	10468138 	
sdd 	0 	0 	0 	0 	7760 	0 	62080 	4911158 	
sde 	0 	0 	0 	0 	4173 	0 	33384 	492852 	
sdf 	0 	0 	0 	0 	4186 	0 	33488 	493994 	
sdg 	0 	0 	0 	0 	2182 	0 	17456 	656424 	
sdh 	0 	0 	0 	0 	2186 	0 	17488 	657295 	
sdi 	0 	0 	0 	0 	21489 	0 	171912 	357830 	
sdj 	0 	0 	0 	0 	21561 	0 	172488 	357311

3.4 randread

3.4.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	12831 	0 	102400 	21037 	0 	0 	0 	10598874 	
sdb 	12831 	0 	102400 	20333 	0 	0 	0 	10599938 	
sdc 	12831 	0 	102400 	19628 	0 	0 	0 	10598546 	
sdd 	12831 	0 	102400 	44931 	0 	0 	0 	5054788 	
sde 	12831 	0 	102400 	78448 	0 	0 	0 	644550 	
sdf 	12831 	0 	102400 	79742 	0 	0 	0 	645831 	
sdg 	12831 	0 	102400 	149941 	0 	0 	0 	810096 	
sdh 	12830 	0 	102400 	150428 	0 	0 	0 	810985 	
sdi 	12831 	0 	102400 	21970 	0 	0 	0 	488195 	
sdj 	12830 	0 	102400 	21449 	0 	0 	0 	487591
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	20535 	0 	164280 	31647 	0 	0 	0 	10598874 	
sdb 	20582 	0 	164656 	31640 	0 	0 	0 	10599938 	
sdc 	20557 	0 	164456 	31666 	0 	0 	0 	10598546 	
sdd 	8930 	0 	71440 	31815 	0 	0 	0 	5054788 	
sde 	5428 	0 	43424 	33241 	0 	0 	0 	644550 	
sdf 	5429 	0 	43432 	33356 	0 	0 	0 	645831 	
sdg 	2729 	0 	21832 	31467 	0 	0 	0 	810096 	
sdh 	2728 	0 	21824 	31475 	0 	0 	0 	810985 	
sdi 	20547 	0 	164376 	31784 	0 	0 	0 	488195 	
sdj 	20535 	0 	164280 	31796 	0 	0 	0 	487591

3.4.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	12815 	0 	102400 	21554 	0 	0 	0 	10598874 	
sdb 	12815 	0 	102400 	21165 	0 	0 	0 	10599938 	
sdc 	12815 	0 	102400 	20645 	0 	0 	0 	10598546 	
sdd 	12815 	0 	102400 	45202 	0 	0 	0 	5054788 	
sde 	12815 	0 	102400 	78168 	0 	0 	0 	644550 	
sdf 	12815 	0 	102400 	77777 	0 	0 	0 	645831 	
sdg 	12815 	0 	102400 	146988 	0 	0 	0 	810096 	
sdh 	12815 	0 	102400 	147223 	0 	0 	0 	810985 	
sdi 	12815 	0 	102400 	20711 	0 	0 	0 	488195 	
sdj 	12815 	0 	102400 	21529 	0 	0 	0 	487591
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	23604 	0 	188832 	31219 	0 	0 	0 	10598874 	
sdb 	23607 	0 	188856 	31204 	0 	0 	0 	10599938 	
sdc 	23370 	0 	186960 	30961 	0 	0 	0 	10598546 	
sdd 	4072 	0 	32576 	13656 	0 	0 	0 	5054788 	
sde 	2025 	0 	16200 	12096 	0 	0 	0 	644550 	
sdf 	2019 	0 	16152 	12099 	0 	0 	0 	645831 	
sdg 	1020 	0 	8160 	11414 	0 	0 	0 	810096 	
sdh 	1017 	0 	8136 	11384 	0 	0 	0 	810985 	
sdi 	23771 	0 	190168 	31420 	0 	0 	0 	488195 	
sdj 	23495 	0 	187960 	31104 	0 	0 	0 	487591

3.4.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21738 	0 	173888 	59188 	0 	0 	0 	10598874 	
sdb 	21650 	0 	173184 	59439 	0 	0 	0 	10599938 	
sdc 	21620 	0 	172944 	59425 	0 	0 	0 	10598546 	
sdd 	7497 	0 	59960 	62036 	0 	0 	0 	5054788 	
sde 	4004 	0 	32016 	68821 	0 	0 	0 	644550 	
sdf 	3998 	0 	31968 	68974 	0 	0 	0 	645831 	
sdg 	2070 	0 	16544 	79930 	0 	0 	0 	810096 	
sdh 	2073 	0 	16568 	79938 	0 	0 	0 	810985 	
sdi 	21617 	0 	172920 	59332 	0 	0 	0 	488195 	
sdj 	21753 	0 	174008 	59245 	0 	0 	0 	487591
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21667 	0 	173336 	59980 	0 	0 	0 	10598874 	
sdb 	21659 	0 	173272 	59792 	0 	0 	0 	10599938 	
sdc 	21633 	0 	173064 	59999 	0 	0 	0 	10598546 	
sdd 	7552 	0 	60416 	62459 	0 	0 	0 	5054788 	
sde 	4039 	0 	32312 	69729 	0 	0 	0 	644550 	
sdf 	4028 	0 	32224 	68405 	0 	0 	0 	645831 	
sdg 	2091 	0 	16728 	80545 	0 	0 	0 	810096 	
sdh 	2093 	0 	16744 	80559 	0 	0 	0 	810985 	
sdi 	21655 	0 	173240 	59994 	0 	0 	0 	488195 	
sdj 	21583 	0 	172664 	60080 	0 	0 	0 	487591

3.4.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randread --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21606 	0 	172840 	123021 	0 	0 	0 	10598874 	
sdb 	21703 	0 	173616 	122969 	0 	0 	0 	10599938 	
sdc 	21723 	0 	173776 	122993 	0 	0 	0 	10598546 	
sdd 	7523 	0 	60176 	134297 	0 	0 	0 	5054788 	
sde 	4024 	0 	32184 	143281 	0 	0 	0 	644550 	
sdf 	4014 	0 	32104 	143135 	0 	0 	0 	645831 	
sdg 	2084 	0 	16664 	145114 	0 	0 	0 	810096 	
sdh 	2080 	0 	16624 	145244 	0 	0 	0 	810985 	
sdi 	21596 	0 	172760 	122924 	0 	0 	0 	488195 	
sdj 	21658 	0 	173256 	122997 	0 	0 	0 	487591
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	21545 	0 	172360 	124562 	0 	0 	0 	10598874 	
sdb 	21665 	0 	173320 	124546 	0 	0 	0 	10599938 	
sdc 	21689 	0 	173512 	124624 	0 	0 	0 	10598546 	
sdd 	7597 	0 	60776 	135796 	0 	0 	0 	5054788 	
sde 	4078 	0 	32624 	145042 	0 	0 	0 	644550 	
sdf 	4051 	0 	32408 	145131 	0 	0 	0 	645831 	
sdg 	2109 	0 	16872 	146996 	0 	0 	0 	810096 	
sdh 	2113 	0 	16904 	147016 	0 	0 	0 	810985 	
sdi 	21581 	0 	172648 	124553 	0 	0 	0 	488195 	
sdj 	21572 	0 	172576 	124647 	0 	0 	0 	487591

3.5 randwrite

3.5.1 iodepth = 1

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=1
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	37 	0 	72 	59 	12800 	0 	102400 	10598874 	
sdb 	38 	0 	112 	51 	12799 	0 	102392 	10599938 	
sdc 	37 	0 	432 	55 	12800 	0 	102400 	10598546 	
sdd 	36 	0 	48 	119 	12801 	0 	102408 	5054788 	
sde 	37 	0 	560 	215 	12800 	0 	102400 	644550 	
sdf 	36 	0 	56 	211 	12801 	0 	102408 	645831 	
sdg 	37 	0 	560 	423 	12799 	0 	102392 	810096 	
sdh 	35 	0 	40 	424 	12801 	0 	102408 	810985 	
sdi 	39 	0 	816 	64 	12799 	0 	102392 	488195 	
sdj 	37 	0 	72 	53 	12800 	0 	102400 	487591
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	20568 	0 	164544 	10619877 	
sdb 	0 	0 	0 	0 	20568 	0 	164544 	10620316 	
sdc 	0 	0 	0 	0 	20552 	0 	164416 	10618256 	
sdd 	0 	0 	0 	0 	8857 	0 	70856 	5099885 	
sde 	0 	0 	0 	0 	5403 	0 	43224 	723109 	
sdf 	0 	0 	0 	0 	5408 	0 	43264 	725546 	
sdg 	0 	0 	0 	0 	2717 	0 	21736 	959382 	
sdh 	0 	0 	0 	0 	2722 	0 	21776 	961023 	
sdi 	0 	0 	0 	0 	20602 	0 	164816 	510230 	
sdj 	0 	0 	0 	0 	20603 	0 	164824 	509156

3.5.2 iodepth = 2

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=2
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	22 	0 	56 	47 	12800 	0 	102400 	10651786 	
sdb 	22 	0 	56 	32 	12800 	0 	102400 	10652227 	
sdc 	21 	0 	48 	35 	12801 	0 	102408 	10650161 	
sdd 	22 	0 	56 	78 	12800 	0 	102400 	5131542 	
sde 	20 	0 	40 	135 	12801 	0 	102408 	756283 	
sdf 	21 	0 	552 	140 	12800 	0 	102400 	758834 	
sdg 	20 	0 	56 	292 	12801 	0 	102408 	990750 	
sdh 	22 	0 	560 	294 	12799 	0 	102392 	992434 	
sdi 	23 	0 	616 	34 	12799 	0 	102392 	542107 	
sdj 	24 	0 	440 	35 	12799 	0 	102392 	541085
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	22634 	0 	181072 	10674040 	
sdb 	0 	0 	0 	0 	25434 	0 	203472 	10674055 	
sdc 	0 	0 	0 	0 	23393 	0 	187144 	10672073 	
sdd 	0 	0 	0 	0 	2351 	0 	18808 	5179141 	
sde 	0 	0 	0 	0 	737 	0 	5896 	839559 	
sdf 	0 	0 	0 	0 	740 	0 	5920 	842211 	
sdg 	0 	0 	0 	0 	672 	0 	5376 	1140181 	
sdh 	0 	0 	0 	0 	673 	0 	5384 	1141844 	
sdi 	0 	0 	0 	0 	26885 	0 	215080 	562914 	
sdj 	0 	0 	0 	0 	24481 	0 	195848 	562103

3.5.3 iodepth = 32

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=32
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	12 	0 	464 	29 	21109 	0 	168872 	10727310 	
sdb 	11 	0 	584 	34 	21130 	0 	169040 	10733497 	
sdc 	7 	0 	48 	18 	21151 	0 	169208 	10727081 	
sdd 	8 	0 	576 	43 	8426 	0 	67408 	5192520 	
sde 	6 	0 	40 	58 	4670 	0 	37360 	844443 	
sdf 	6 	0 	40 	47 	4694 	0 	37552 	847086 	
sdg 	6 	0 	40 	117 	2521 	0 	20168 	1148137 	
sdh 	5 	0 	32 	98 	2517 	0 	20136 	1149838 	
sdi 	7 	0 	48 	17 	20900 	0 	167200 	625835 	
sdj 	8 	0 	608 	30 	20882 	0 	167056 	619669
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	24076 	0 	192608 	10801471 	
sdb 	0 	0 	0 	0 	24012 	0 	192096 	10807535 	
sdc 	0 	0 	0 	0 	23935 	0 	191480 	10801185 	
sdd 	0 	0 	0 	0 	6239 	0 	49912 	5271142 	
sde 	0 	0 	0 	0 	523 	0 	4184 	927677 	
sdf 	0 	0 	0 	0 	599 	0 	4792 	929891 	
sdg 	0 	0 	0 	0 	145 	0 	1160 	1250168 	
sdh 	0 	0 	0 	0 	145 	0 	1160 	1252170 	
sdi 	0 	0 	0 	0 	24154 	0 	193232 	700051 	
sdj 	0 	0 	0 	0 	24172 	0 	193376 	693923

3.5.4 iodepth = 64

fio --ioengine=libaio --size=500M --bs=4k --direct=1 --numjobs 1 --filename=/dev/mapper/mpatha --name=bla --rw=randwrite --iodepth=64
  1. ST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 0:0:0:1 sda 8:0   active undef running
  |- 1:0:0:1 sdb 8:16  active undef running
  |- 2:0:0:1 sdc 8:32  active undef running
  |- 3:0:0:1 sdd 8:48  active undef running
  |- 4:0:0:1 sde 8:64  active undef running
  |- 5:0:0:1 sdf 8:80  active undef running
  |- 6:0:0:1 sdg 8:96  active undef running
  |- 7:0:0:1 sdh 8:112 active undef running
  |- 8:0:0:1 sdi 8:128 active undef running
  `- 9:0:0:1 sdj 8:144 active undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	6 	0 	40 	14 	21007 	0 	168056 	10975814 	
sdb 	7 	0 	552 	19 	20962 	0 	167696 	10981999 	
sdc 	10 	0 	120 	45 	21036 	0 	168288 	10975026 	
sdd 	6 	0 	544 	41 	8610 	0 	68880 	5313493 	
sde 	6 	0 	544 	78 	4811 	0 	38488 	932384 	
sdf 	8 	0 	56 	162 	4835 	0 	38680 	935152 	
sdg 	6 	0 	40 	149 	2588 	0 	20704 	1252453 	
sdh 	5 	0 	32 	122 	2579 	0 	20632 	1254474 	
sdi 	8 	0 	72 	31 	20788 	0 	166304 	875764 	
sdj 	7 	0 	424 	20 	20784 	0 	166272 	868920
  2. HST
    mpatha (360000000000000000e00000000010001) dm-0 IET,VIRTUAL-DISK
size=2.0G features='0' hwhandler='0' wp=rw
`-+- policy='historical-service-time 2' prio=0 status=active
  |- 0:0:0:1 sda 8:0   undef undef running
  |- 1:0:0:1 sdb 8:16  undef undef running
  |- 2:0:0:1 sdc 8:32  undef undef running
  |- 3:0:0:1 sdd 8:48  undef undef running
  |- 4:0:0:1 sde 8:64  undef undef running
  |- 5:0:0:1 sdf 8:80  undef undef running
  |- 6:0:0:1 sdg 8:96  undef undef running
  |- 7:0:0:1 sdh 8:112 undef undef running
  |- 8:0:0:1 sdi 8:128 undef undef running
  `- 9:0:0:1 sdj 8:144 undef undef running
dev	reads	rmerg	secR	tRed	wrts	wmerg	secW	tWrit
sda 	0 	0 	0 	0 	21746 	0 	173968 	11135898 	
sdb 	0 	0 	0 	0 	21869 	0 	174952 	11142135 	
sdc 	0 	0 	0 	0 	21874 	0 	174992 	11135022 	
sdd 	0 	0 	0 	0 	8682 	0 	69456 	5482839 	
sde 	0 	0 	0 	0 	4211 	0 	33688 	1119844 	
sdf 	0 	0 	0 	0 	4126 	0 	33008 	1122180 	
sdg 	0 	0 	0 	0 	825 	0 	6600 	1443617 	
sdh 	0 	0 	0 	0 	831 	0 	6648 	1445842 	
sdi 	0 	0 	0 	0 	22070 	0 	176560 	1035899 	
sdj 	0 	0 	0 	0 	21766 	0 	174128 	1029505

Author: Gabriel Krisman Bertazi

Created: 2020-04-16 Thu 13:28

Validate