[root@iotest ~]# cat fiotester2.sh 
#!/bin/bash

FIO="/usr/bin/fio"
FN="/dev/sdb"
OFFSET="1794597978112"
# 5GB are 5.368.709.120 Bytes
DEVSIZE="1799966687232"
DEVSIZE10P="179996668723"
BSIOPS="512"
BSTHR="1m"

for TESTCASE in "read" "write"
do 
	for NJ in 1 4
	do
		echo "#################################################################################"
		echo "Executing $TESTCASE tests with numjobs=$NJ"
		for testrun in `seq 1 10`
		do
			case $testrun in
				1)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				2)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				3)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				4)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				5)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				6)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				7)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				8)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				9)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				10)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
			esac
			echo "---------------------------------------------------------------------------------"
			echo "Executing test $testrun: $TESTCMD"
			$TESTCMD
			echo "Sleeping for 30 seconds... ------------------------------------------------------"
			sleep 30
		done
	done
done
[root@iotest ~]# ./fiotester2.sh 
#################################################################################
Executing read tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [17M/0K /s] [34K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25058
  read : io=966MB, bw=16,480KB/s, iops=32,960, runt= 60000msec
    clat (usec): min=18, max=9,409, avg=28.43, stdev=15.91
    bw (KB/s) : min=13764, max=16952, per=99.98%, avg=16477.50, stdev=498.71
  cpu          : usr=10.08%, sys=25.05%, ctx=1978249, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=1977645/0, short=0/0
     lat (usec): 20=0.07%, 50=98.96%, 100=0.93%, 250=0.02%, 500=0.02%
     lat (usec): 750=0.01%, 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=966MB, aggrb=16,480KB/s, minb=16,875KB/s, maxb=16,875KB/s, mint=60000msec, maxt=60000msec

Disk stats (read/write):
  sdb: ios=1974681/0, merge=0/0, ticks=26147/0, in_queue=26254, util=43.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=179996668723 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [17M/0K /s] [33K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25063
  read : io=966MB, bw=16,491KB/s, iops=32,981, runt= 60001msec
    clat (usec): min=18, max=10,783, avg=28.43, stdev=12.67
    bw (KB/s) : min=11958, max=16922, per=100.01%, avg=16490.87, stdev=731.28
  cpu          : usr=9.15%, sys=22.88%, ctx=1979549, majf=0, minf=27
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=1978917/0, short=0/0
     lat (usec): 20=0.05%, 50=98.90%, 100=0.88%, 250=0.14%, 500=0.02%
     lat (usec): 750=0.01%, 1000=0.01%
     lat (msec): 2=0.01%, 10=0.01%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=966MB, aggrb=16,490KB/s, minb=16,886KB/s, maxb=16,886KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=1975818/0, merge=0/0, ticks=25643/0, in_queue=25471, util=42.56%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [90K/0K /s] [176/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25068
  read : io=5,357KB, bw=91,405B/s, iops=178, runt= 60008msec
    clat (usec): min=106, max=15,644, avg=5589.71, stdev=1684.52
    bw (KB/s) : min=   75, max=   96, per=99.88%, avg=88.89, stdev= 3.29
  cpu          : usr=0.21%, sys=0.35%, ctx=10733, majf=0, minf=69
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=10713/0, short=0/0
     lat (usec): 250=0.02%
     lat (msec): 2=0.71%, 4=17.65%, 10=81.39%, 20=0.23%

Run status group 0 (all jobs):
   READ: io=5,356KB, aggrb=89KB/s, minb=91KB/s, maxb=91KB/s, mint=60008msec, maxt=60008msec

Disk stats (read/write):
  sdb: ios=10692/0, merge=0/0, ticks=59613/0, in_queue=59603, util=99.20%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=179996668723 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [138K/0K /s] [270/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25073
  read : io=7,985KB, bw=133KB/s, iops=266, runt= 60001msec
    clat (usec): min=99, max=42,596, avg=3746.98, stdev=1291.34
    bw (KB/s) : min=  117, max=  140, per=99.83%, avg=132.77, stdev= 4.06
  cpu          : usr=0.34%, sys=0.59%, ctx=15992, majf=0, minf=1606
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15969/0, short=0/0
     lat (usec): 100=0.01%, 250=0.07%, 750=0.01%, 1000=0.01%
     lat (msec): 2=7.51%, 4=49.13%, 10=43.20%, 20=0.04%, 50=0.02%

Run status group 0 (all jobs):
   READ: io=7,984KB, aggrb=133KB/s, minb=136KB/s, maxb=136KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=15938/0, merge=0/0, ticks=59602/0, in_queue=59592, util=99.22%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [573M/0K /s] [559/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25080
  read : io=5,120MB, bw=553MB/s, iops=553, runt=  9254msec
    clat (usec): min=478, max=22,928, avg=1804.97, stdev=1540.91
    bw (KB/s) : min=550756, max=585434, per=100.20%, avg=567685.11, stdev=10679.67
  cpu          : usr=0.15%, sys=5.90%, ctx=5149, majf=0, minf=279
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0
     lat (usec): 500=0.33%, 750=21.89%, 1000=41.11%
     lat (msec): 2=1.88%, 4=26.86%, 10=7.83%, 20=0.08%, 50=0.02%

Run status group 0 (all jobs):
   READ: io=5,120MB, aggrb=553MB/s, minb=567MB/s, maxb=567MB/s, mint=9254msec, maxt=9254msec

Disk stats (read/write):
  sdb: ios=40722/0, merge=0/0, ticks=40205/0, in_queue=40206, util=97.15%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [346M/0K /s] [337/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25083
  read : io=5,120MB, bw=335MB/s, iops=335, runt= 15268msec
    clat (usec): min=484, max=59,444, avg=2979.38, stdev=3149.43
    bw (KB/s) : min=298919, max=354304, per=100.05%, avg=343549.37, stdev=10673.18
  cpu          : usr=0.18%, sys=3.88%, ctx=5153, majf=0, minf=279
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0
     lat (usec): 500=0.29%, 750=26.82%, 1000=33.11%
     lat (msec): 2=2.71%, 4=2.01%, 10=34.79%, 20=0.23%, 50=0.02%
     lat (msec): 100=0.02%

Run status group 0 (all jobs):
   READ: io=5,120MB, aggrb=335MB/s, minb=343MB/s, maxb=343MB/s, mint=15268msec, maxt=15268msec

Disk stats (read/write):
  sdb: ios=40813/0, merge=0/0, ticks=64627/0, in_queue=64636, util=98.29%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [564M/0K /s] [550/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25088
  read : io=33,265MB, bw=554MB/s, iops=554, runt= 60001msec
    clat (usec): min=495, max=23,972, avg=1801.26, stdev=1566.82
    bw (KB/s) : min=529304, max=587776, per=100.10%, avg=568299.13, stdev=12014.18
  cpu          : usr=0.24%, sys=6.20%, ctx=33464, majf=0, minf=282
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=33265/0, short=0/0
     lat (usec): 500=0.01%, 750=22.32%, 1000=43.43%
     lat (msec): 2=0.51%, 4=25.74%, 10=7.88%, 20=0.13%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=33,265MB, aggrb=554MB/s, minb=568MB/s, maxb=568MB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=265692/0, merge=0/0, ticks=255872/0, in_queue=255908, util=97.78%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [101M/0K /s] [98/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25094
  read : io=4,845MB, bw=82,677KB/s, iops=80, runt= 60008msec
    clat (usec): min=481, max=32,672, avg=12380.21, stdev=4061.89
    bw (KB/s) : min=71111, max=122147, per=100.08%, avg=82740.82, stdev=7796.67
  cpu          : usr=0.04%, sys=1.00%, ctx=4894, majf=0, minf=279
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=4845/0, short=0/0
     lat (usec): 500=0.23%, 750=3.59%, 1000=0.72%
     lat (msec): 2=0.29%, 4=0.35%, 10=15.73%, 20=78.16%, 50=0.93%

Run status group 0 (all jobs):
   READ: io=4,845MB, aggrb=82,676KB/s, minb=84,661KB/s, maxb=84,661KB/s, mint=60008msec, maxt=60008msec

Disk stats (read/write):
  sdb: ios=38688/0, merge=0/0, ticks=386343/0, in_queue=386342, util=99.74%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [74M/0K /s] [71/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25128
  read : io=3,834MB, bw=65,428KB/s, iops=63, runt= 60005msec
    clat (usec): min=502, max=55,759, avg=15645.25, stdev=4196.89
    bw (KB/s) : min=58803, max=74436, per=100.08%, avg=65480.35, stdev=3611.55
  cpu          : usr=0.03%, sys=0.80%, ctx=3865, majf=0, minf=283
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3834/0, short=0/0
     lat (usec): 750=2.14%, 1000=0.70%
     lat (msec): 2=0.13%, 4=0.03%, 10=4.23%, 20=86.96%, 50=5.79%
     lat (msec): 100=0.03%

Run status group 0 (all jobs):
   READ: io=3,834MB, aggrb=65,428KB/s, minb=66,998KB/s, maxb=66,998KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdb: ios=30620/0, merge=0/0, ticks=389565/0, in_queue=389586, util=99.82%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [68M/0K /s] [66/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25133
  read : io=3,991MB, bw=68,107KB/s, iops=66, runt= 60005msec
    clat (msec): min=6, max=105, avg=15.03, stdev= 4.20
    bw (KB/s) : min=60014, max=79238, per=100.04%, avg=68131.82, stdev=3602.99
  cpu          : usr=0.04%, sys=0.89%, ctx=4006, majf=0, minf=331
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3991/0, short=0/0

     lat (msec): 10=6.34%, 20=84.54%, 50=9.10%, 250=0.03%

Run status group 0 (all jobs):
   READ: io=3,991MB, aggrb=68,107KB/s, minb=69,741KB/s, maxb=69,741KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdb: ios=31874/0, merge=0/0, ticks=388143/0, in_queue=388175, util=99.81%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing read tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [18M/0K /s] [36K/0 iops] [eta 00m:00s] 
file1: (groupid=0, jobs=4): err= 0: pid=25141
  read : io=1,349MB, bw=23,031KB/s, iops=46,061, runt= 60001msec
    clat (usec): min=18, max=307K, avg=98.23, stdev=1550.89
    bw (KB/s) : min=    4, max=15596, per=22.54%, avg=5190.20, stdev=1762.54
  cpu          : usr=2.90%, sys=11.58%, ctx=2807082, majf=0, minf=102
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2763737/0, short=0/0
     lat (usec): 20=0.01%, 50=85.11%, 100=10.98%, 250=2.52%, 500=1.35%
     lat (usec): 750=0.01%, 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
     lat (msec): 100=0.01%, 250=0.02%, 500=0.01%

Run status group 0 (all jobs):
   READ: io=1,349MB, aggrb=23,030KB/s, minb=23,583KB/s, maxb=23,583KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=2760025/0, merge=65/0, ticks=190188/0, in_queue=190587, util=99.52%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=179996668723 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [23M/0K /s] [45K/0 iops] [eta 00m:00s]  
file1: (groupid=0, jobs=4): err= 0: pid=25149
  read : io=1,353MB, bw=23,095KB/s, iops=46,190, runt= 60001msec
    clat (usec): min=18, max=204K, avg=84.95, stdev=962.66
    bw (KB/s) : min= 1128, max=13343, per=24.89%, avg=5748.29, stdev=1244.22
  cpu          : usr=3.18%, sys=10.12%, ctx=2979536, majf=0, minf=97
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2771502/0, short=0/0
     lat (usec): 20=0.01%, 50=83.39%, 100=12.56%, 250=3.03%, 500=0.97%
     lat (usec): 750=0.01%, 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
     lat (msec): 100=0.01%, 250=0.04%

Run status group 0 (all jobs):
   READ: io=1,353MB, aggrb=23,095KB/s, minb=23,649KB/s, maxb=23,649KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=2767305/0, merge=29/0, ticks=184477/0, in_queue=184640, util=99.55%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [252K/0K /s] [492/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25162
  read : io=14,884KB, bw=248KB/s, iops=496, runt= 60011msec
    clat (usec): min=48, max=52,051, avg=8047.97, stdev=2146.34
    bw (KB/s) : min=   48, max=   72, per=24.81%, avg=61.53, stdev= 2.11
  cpu          : usr=0.22%, sys=0.23%, ctx=29825, majf=0, minf=500
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=29768/0, short=0/0
     lat (usec): 50=0.01%, 100=0.01%, 250=0.01%, 750=0.01%
     lat (msec): 2=0.39%, 4=10.19%, 10=66.03%, 20=21.12%, 50=2.22%
     lat (msec): 100=0.01%

Run status group 0 (all jobs):
   READ: io=14,884KB, aggrb=248KB/s, minb=253KB/s, maxb=253KB/s, mint=60011msec, maxt=60011msec

Disk stats (read/write):
  sdb: ios=29724/0, merge=0/0, ticks=238930/0, in_queue=238913, util=99.48%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=179996668723 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 1 (f=1): [___r] [25.6% done] [255K/0K /s] [499/0 iops] [eta 03m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25170
  read : io=22,441KB, bw=374KB/s, iops=747, runt= 60007msec
    clat (usec): min=63, max=36,294, avg=5356.80, stdev=1524.92
    bw (KB/s) : min=   67, max=  120, per=24.86%, avg=92.73, stdev= 3.20
  cpu          : usr=0.25%, sys=0.28%, ctx=44946, majf=0, minf=1659
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=44881/0, short=0/0
     lat (usec): 100=0.05%, 250=0.02%, 750=0.01%, 1000=0.02%
     lat (msec): 2=4.07%, 4=32.49%, 10=56.37%, 20=6.57%, 50=0.39%

Run status group 0 (all jobs):
   READ: io=22,440KB, aggrb=373KB/s, minb=382KB/s, maxb=382KB/s, mint=60007msec, maxt=60007msec

Disk stats (read/write):
  sdb: ios=44876/0, merge=0/0, ticks=239221/0, in_queue=239225, util=99.48%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [2,281M/0K /s] [2K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25178
  read : io=20,480MB, bw=2,106MB/s, iops=2,106, runt=  9723msec
    clat (usec): min=474, max=279K, avg=1895.33, stdev=1433.24
    bw (KB/s) : min=314159, max=579584, per=25.00%, avg=539151.98, stdev=28272.97
  cpu          : usr=0.21%, sys=7.56%, ctx=20665, majf=0, minf=1116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=20480/0, short=0/0
     lat (usec): 500=0.03%, 750=1.07%, 1000=1.39%
     lat (msec): 2=79.11%, 4=16.99%, 10=1.06%, 20=0.24%, 50=0.06%
     lat (msec): 100=0.01%, 250=0.02%, 500=0.01%

Run status group 0 (all jobs):
   READ: io=20,480MB, aggrb=2,106MB/s, minb=2,157MB/s, maxb=2,157MB/s, mint=9723msec, maxt=9723msec

Disk stats (read/write):
  sdb: ios=163192/0, merge=0/0, ticks=244508/0, in_queue=244573, util=99.26%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [777M/0K /s] [758/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25186
  read : io=20,480MB, bw=1,176MB/s, iops=1,175, runt= 17419msec
    clat (usec): min=471, max=143K, avg=3398.00, stdev=2029.12
    bw (KB/s) : min=130549, max=354304, per=25.28%, avg=304342.06, stdev=32155.48
  cpu          : usr=0.13%, sys=4.53%, ctx=20661, majf=0, minf=1120
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=20480/0, short=0/0
     lat (usec): 500=0.50%, 750=1.38%, 1000=4.13%
     lat (msec): 2=42.14%, 4=21.82%, 10=26.96%, 20=2.07%, 50=0.97%
     lat (msec): 100=0.01%, 250=0.02%

Run status group 0 (all jobs):
   READ: io=20,480MB, aggrb=1,176MB/s, minb=1,204MB/s, maxb=1,204MB/s, mint=17419msec, maxt=17419msec

Disk stats (read/write):
  sdb: ios=162912/0, merge=0/0, ticks=354016/0, in_queue=354072, util=99.68%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [745M/0K /s] [727/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25194
  read : io=51,196MB, bw=853MB/s, iops=853, runt= 60009msec
    clat (usec): min=473, max=584K, avg=4631.91, stdev=3188.82
    bw (KB/s) : min=69493, max=580198, per=25.37%, avg=221670.78, stdev=50535.51
  cpu          : usr=0.09%, sys=3.12%, ctx=51634, majf=0, minf=1121
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=51196/0, short=0/0
     lat (usec): 500=0.76%, 750=4.35%, 1000=7.03%
     lat (msec): 2=41.77%, 4=12.89%, 10=17.86%, 20=13.24%, 50=2.07%
     lat (msec): 100=0.01%, 250=0.02%, 500=0.01%, 750=0.01%

Run status group 0 (all jobs):
   READ: io=51,196MB, aggrb=853MB/s, minb=874MB/s, maxb=874MB/s, mint=60009msec, maxt=60009msec

Disk stats (read/write):
  sdb: ios=409050/0, merge=0/0, ticks=1247540/0, in_queue=1247632, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [133M/0K /s] [129/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25202
  read : io=7,604MB, bw=127MB/s, iops=126, runt= 60027msec
    clat (usec): min=501, max=113K, avg=31900.15, stdev=5918.69
    bw (KB/s) : min=23316, max=43828, per=24.79%, avg=32154.11, stdev=1539.32
  cpu          : usr=0.02%, sys=0.39%, ctx=7652, majf=0, minf=1124
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7604/0, short=0/0
     lat (usec): 750=1.67%, 1000=0.11%
     lat (msec): 2=0.03%, 4=0.01%, 10=1.09%, 20=11.03%, 50=80.34%
     lat (msec): 100=5.69%, 250=0.03%

Run status group 0 (all jobs):
   READ: io=7,604MB, aggrb=127MB/s, minb=130MB/s, maxb=130MB/s, mint=60027msec, maxt=60027msec

Disk stats (read/write):
  sdb: ios=60718/0, merge=0/0, ticks=1382899/0, in_queue=1383172, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [96M/0K /s] [93/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25239
  read : io=5,413MB, bw=92,316KB/s, iops=90, runt= 60043msec
    clat (usec): min=514, max=158K, avg=44001.77, stdev=8167.92
    bw (KB/s) : min=16282, max=31267, per=25.23%, avg=23294.78, stdev=1186.05
  cpu          : usr=0.01%, sys=0.29%, ctx=5443, majf=0, minf=1116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5413/0, short=0/0
     lat (usec): 750=1.66%, 1000=0.07%
     lat (msec): 10=0.39%, 20=3.84%, 50=61.37%, 100=32.24%, 250=0.42%

Run status group 0 (all jobs):
   READ: io=5,413MB, aggrb=92,315KB/s, minb=94,531KB/s, maxb=94,531KB/s, mint=60043msec, maxt=60043msec

Disk stats (read/write):
  sdb: ios=43213/0, merge=0/0, ticks=1358179/0, in_queue=1358892, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [81M/0K /s] [79/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25247
  read : io=4,838MB, bw=82,519KB/s, iops=80, runt= 60036msec
    clat (usec): min=597, max=137K, avg=49742.97, stdev=7761.58
    bw (KB/s) : min=15255, max=26101, per=25.00%, avg=20630.64, stdev=1010.23
  cpu          : usr=0.01%, sys=0.25%, ctx=4871, majf=0, minf=1884
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=4838/0, short=0/0
     lat (usec): 750=0.02%
     lat (msec): 20=1.41%, 50=54.28%, 100=43.68%, 250=0.62%

Run status group 0 (all jobs):
   READ: io=4,838MB, aggrb=82,519KB/s, minb=84,499KB/s, maxb=84,499KB/s, mint=60036msec, maxt=60036msec

Disk stats (read/write):
  sdb: ios=38629/0, merge=0/0, ticks=1359680/0, in_queue=1360290, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/16M /s] [0/31K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25256
  write: io=922MB, bw=15,739KB/s, iops=31,477, runt= 60001msec
    clat (usec): min=20, max=53,648, avg=29.77, stdev=39.70
    bw (KB/s) : min=14185, max=16082, per=100.02%, avg=15740.95, stdev=261.37
  cpu          : usr=9.93%, sys=20.98%, ctx=1889301, majf=0, minf=23
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1888686, short=0/0
     lat (usec): 50=98.98%, 100=0.91%, 250=0.11%, 500=0.01%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 100=0.01%

Run status group 0 (all jobs):
  WRITE: io=922MB, aggrb=15,738KB/s, minb=16,116KB/s, maxb=16,116KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=0/1885898, merge=0/0, ticks=0/25047, in_queue=26190, util=43.76%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=179996668723 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/16M /s] [0/32K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25263
  write: io=925MB, bw=15,789KB/s, iops=31,577, runt= 60001msec
    clat (usec): min=19, max=1,768, avg=29.68, stdev= 7.80
    bw (KB/s) : min=14400, max=16062, per=100.00%, avg=15788.66, stdev=233.83
  cpu          : usr=11.59%, sys=22.23%, ctx=1895363, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1894681, short=0/0
     lat (usec): 20=0.01%, 50=98.93%, 100=0.95%, 250=0.12%, 500=0.01%
     lat (usec): 750=0.01%, 1000=0.01%
     lat (msec): 2=0.01%

Run status group 0 (all jobs):
  WRITE: io=925MB, aggrb=15,788KB/s, minb=16,167KB/s, maxb=16,167KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=0/1891917, merge=0/0, ticks=0/21733, in_queue=22085, util=36.90%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/229K /s] [0/447 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25268
  write: io=11,221KB, bw=187KB/s, iops=373, runt= 60044msec
    clat (usec): min=28, max=1,145K, avg=2668.81, stdev=23701.04
    bw (KB/s) : min=   34, max=  942, per=101.70%, avg=189.15, stdev=159.22
  cpu          : usr=0.26%, sys=0.36%, ctx=22454, majf=0, minf=168
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/22441, short=0/0
     lat (usec): 50=43.06%, 100=5.48%, 250=31.30%, 500=17.02%, 750=0.07%
     lat (usec): 1000=0.02%
     lat (msec): 2=0.06%, 10=0.01%, 20=0.07%, 50=1.52%, 100=0.74%
     lat (msec): 250=0.47%, 500=0.15%, 750=0.01%, 1000=0.01%, 2000=0.01%

Run status group 0 (all jobs):
  WRITE: io=11,220KB, aggrb=186KB/s, minb=191KB/s, maxb=191KB/s, mint=60044msec, maxt=60044msec

Disk stats (read/write):
  sdb: ios=0/22311, merge=0/0, ticks=0/59367, in_queue=59419, util=98.92%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=179996668723 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/290K /s] [0/568 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25273
  write: io=19,390KB, bw=323KB/s, iops=646, runt= 60001msec
    clat (usec): min=28, max=1,022K, avg=1542.13, stdev=18317.08
    bw (KB/s) : min=   64, max= 1807, per=106.13%, avg=342.80, stdev=203.53
  cpu          : usr=0.39%, sys=0.60%, ctx=38813, majf=0, minf=1843
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/38780, short=0/0
     lat (usec): 50=30.08%, 100=5.10%, 250=28.82%, 500=34.31%, 750=0.27%
     lat (usec): 1000=0.05%
     lat (msec): 2=0.03%, 4=0.01%, 10=0.04%, 20=0.04%, 50=0.15%
     lat (msec): 100=0.77%, 250=0.26%, 500=0.03%, 750=0.01%, 1000=0.02%
     lat (msec): 2000=0.01%

Run status group 0 (all jobs):
  WRITE: io=19,390KB, aggrb=323KB/s, minb=330KB/s, maxb=330KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=0/38695, merge=0/0, ticks=0/59122, in_queue=59252, util=98.60%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/579M /s] [0/565 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25278
  write: io=5,120MB, bw=573MB/s, iops=572, runt=  8938msec
    clat (usec): min=438, max=43,906, avg=1742.96, stdev=2284.27
    bw (KB/s) : min=530432, max=766467, per=99.93%, avg=586183.94, stdev=48375.93
  cpu          : usr=0.28%, sys=5.58%, ctx=5143, majf=0, minf=22
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0
     lat (usec): 500=1.52%, 750=0.12%, 1000=5.94%
     lat (msec): 2=86.60%, 4=0.02%, 10=1.76%, 20=4.02%, 50=0.02%

Run status group 0 (all jobs):
  WRITE: io=5,120MB, aggrb=573MB/s, minb=587MB/s, maxb=587MB/s, mint=8938msec, maxt=8938msec

Disk stats (read/write):
  sdb: ios=0/39825, merge=0/0, ticks=0/40490, in_queue=40510, util=97.75%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/346M /s] [0/337 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25283
  write: io=5,120MB, bw=344MB/s, iops=344, runt= 14871msec
    clat (usec): min=418, max=37,318, avg=2901.79, stdev=7045.48
    bw (KB/s) : min=323779, max=558545, per=100.20%, avg=353251.68, stdev=41891.54
  cpu          : usr=0.16%, sys=3.83%, ctx=5145, majf=0, minf=22
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0
     lat (usec): 500=1.50%, 750=0.31%, 1000=13.61%
     lat (msec): 2=78.65%, 4=0.04%, 50=5.88%

Run status group 0 (all jobs):
  WRITE: io=5,120MB, aggrb=344MB/s, minb=353MB/s, maxb=353MB/s, mint=14871msec, maxt=14871msec

Disk stats (read/write):
  sdb: ios=0/40527, merge=0/0, ticks=0/70492, in_queue=70493, util=98.78%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/577M /s] [0/563 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25286
  write: io=33,936MB, bw=566MB/s, iops=565, runt= 60006msec
    clat (usec): min=431, max=19,913, avg=1765.65, stdev=2229.32
    bw (KB/s) : min=557056, max=778240, per=100.09%, avg=579646.96, stdev=19675.16
  cpu          : usr=0.23%, sys=5.88%, ctx=34151, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/33936, short=0/0
     lat (usec): 500=0.23%, 750=0.10%, 1000=4.21%
     lat (msec): 2=89.43%, 4=0.01%, 10=2.15%, 20=3.87%

Run status group 0 (all jobs):
  WRITE: io=33,936MB, aggrb=566MB/s, minb=579MB/s, maxb=579MB/s, mint=60006msec, maxt=60006msec

Disk stats (read/write):
  sdb: ios=0/271066, merge=0/0, ticks=0/277276, in_queue=277275, util=98.58%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/199M /s] [0/194 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25322
  write: io=5,120MB, bw=193MB/s, iops=192, runt= 26550msec
    clat (usec): min=426, max=206K, avg=5181.37, stdev=16610.51
    bw (KB/s) : min=135168, max=488495, per=100.10%, avg=197670.37, stdev=47219.91
  cpu          : usr=0.09%, sys=1.79%, ctx=5146, majf=0, minf=20
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0
     lat (usec): 500=1.52%, 750=0.18%, 1000=9.12%
     lat (msec): 2=75.33%, 4=7.93%, 10=0.20%, 20=0.21%, 50=1.21%
     lat (msec): 100=3.73%, 250=0.57%

Run status group 0 (all jobs):
  WRITE: io=5,120MB, aggrb=193MB/s, minb=197MB/s, maxb=197MB/s, mint=26550msec, maxt=26550msec

Disk stats (read/write):
  sdb: ios=0/40923, merge=0/0, ticks=0/106722, in_queue=107061, util=99.23%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/157M /s] [0/153 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=25325
  write: io=5,120MB, bw=142MB/s, iops=141, runt= 36121msec
    clat (usec): min=434, max=249K, avg=7050.61, stdev=24465.94
    bw (KB/s) : min=93423, max=460800, per=100.62%, avg=146053.91, stdev=43957.82
  cpu          : usr=0.06%, sys=1.33%, ctx=5139, majf=0, minf=22
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0
     lat (usec): 500=1.52%, 750=0.16%, 1000=12.11%
     lat (msec): 2=75.55%, 4=4.73%, 10=0.02%, 20=0.08%, 50=0.35%
     lat (msec): 100=2.91%, 250=2.58%

Run status group 0 (all jobs):
  WRITE: io=5,120MB, aggrb=142MB/s, minb=145MB/s, maxb=145MB/s, mint=36121msec, maxt=36121msec

Disk stats (read/write):
  sdb: ios=0/40885, merge=0/0, ticks=0/149320, in_queue=149556, util=99.58%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=1799966687232 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/99M /s] [0/96 iops] [eta 00m:00s] 
file1: (groupid=0, jobs=1): err= 0: pid=25330
  write: io=6,724MB, bw=112MB/s, iops=111, runt= 60100msec
    clat (usec): min=427, max=260K, avg=8934.10, stdev=31841.37
    bw (KB/s) : min=83367, max=389039, per=100.80%, avg=115480.72, stdev=29356.82
  cpu          : usr=0.06%, sys=1.11%, ctx=6762, majf=0, minf=74
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/6724, short=0/0
     lat (usec): 500=1.16%, 750=0.33%, 1000=12.66%
     lat (msec): 2=75.91%, 4=3.90%, 20=0.01%, 50=0.13%, 100=1.52%
     lat (msec): 250=4.37%, 500=0.01%

Run status group 0 (all jobs):
  WRITE: io=6,724MB, aggrb=112MB/s, minb=115MB/s, maxb=115MB/s, mint=60100msec, maxt=60100msec

Disk stats (read/write):
  sdb: ios=0/53659, merge=0/0, ticks=0/230971, in_queue=231432, util=99.81%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/25M /s] [0/50K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25336
  write: io=1,413MB, bw=24,116KB/s, iops=48,231, runt= 60001msec
    clat (usec): min=22, max=4,088, avg=80.79, stdev=19.24
    bw (KB/s) : min= 5345, max= 6635, per=24.99%, avg=6026.00, stdev=110.20
  cpu          : usr=3.62%, sys=8.71%, ctx=3230406, majf=0, minf=103
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2893948, short=0/0
     lat (usec): 50=28.19%, 100=45.99%, 250=25.75%, 500=0.06%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
  WRITE: io=1,413MB, aggrb=24,115KB/s, minb=24,694KB/s, maxb=24,694KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=0/2627039, merge=0/261572, ticks=0/166781, in_queue=167149, util=99.97%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=179996668723 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/25M /s] [0/50K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25346
  write: io=1,423MB, bw=24,288KB/s, iops=48,576, runt= 60001msec
    clat (usec): min=22, max=4,409, avg=80.18, stdev=18.01
    bw (KB/s) : min= 5365, max= 6655, per=25.00%, avg=6071.19, stdev=103.76
  cpu          : usr=3.59%, sys=8.82%, ctx=3263887, majf=0, minf=93
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2914665, short=0/0
     lat (usec): 50=28.25%, 100=46.50%, 250=25.18%, 500=0.06%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
  WRITE: io=1,423MB, aggrb=24,288KB/s, minb=24,871KB/s, maxb=24,871KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=0/2634771, merge=0/274752, ticks=0/164004, in_queue=164226, util=99.99%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/228K /s] [0/445 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25354
  write: io=14,476KB, bw=241KB/s, iops=482, runt= 60028msec
    clat (usec): min=28, max=1,561K, avg=7415.34, stdev=24297.99
    bw (KB/s) : min=    6, max= 1425, per=28.56%, avg=68.83, stdev=62.64
  cpu          : usr=0.19%, sys=0.10%, ctx=29327, majf=0, minf=459
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/28951, short=0/0
     lat (usec): 50=5.54%, 100=14.24%, 250=29.88%, 500=10.61%, 750=6.61%
     lat (usec): 1000=7.22%
     lat (msec): 2=20.67%, 4=0.14%, 10=0.01%, 20=0.03%, 50=0.34%
     lat (msec): 100=1.97%, 250=2.03%, 500=0.54%, 750=0.10%, 1000=0.06%
     lat (msec): 2000=0.01%

Run status group 0 (all jobs):
  WRITE: io=14,475KB, aggrb=241KB/s, minb=246KB/s, maxb=246KB/s, mint=60028msec, maxt=60028msec

Disk stats (read/write):
  sdb: ios=0/28886, merge=0/0, ticks=0/239005, in_queue=239092, util=99.46%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=179996668723 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 1 (f=1): [__w_] [25.7% done] [0K/327K /s] [0/640 iops] [eta 02m:59s] 
file1: (groupid=0, jobs=4): err= 0: pid=25365
  write: io=19,487KB, bw=324KB/s, iops=648, runt= 60062msec
    clat (usec): min=28, max=527K, avg=6417.16, stdev=12020.18
    bw (KB/s) : min=   10, max= 1469, per=24.05%, avg=77.94, stdev=16.29
  cpu          : usr=0.19%, sys=0.12%, ctx=39221, majf=0, minf=1482
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/38973, short=0/0
     lat (usec): 50=3.36%, 100=4.81%, 250=9.83%, 500=6.92%, 750=10.48%
     lat (usec): 1000=13.31%
     lat (msec): 2=45.10%, 4=0.53%, 10=0.05%, 20=0.03%, 50=0.16%
     lat (msec): 100=3.82%, 250=1.52%, 500=0.08%, 750=0.01%

Run status group 0 (all jobs):
  WRITE: io=19,486KB, aggrb=324KB/s, minb=332KB/s, maxb=332KB/s, mint=60062msec, maxt=60062msec

Disk stats (read/write):
  sdb: ios=0/38972, merge=0/0, ticks=0/239307, in_queue=239390, util=99.42%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/1,651M /s] [0/2K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25373
  write: io=20,480MB, bw=1,619MB/s, iops=1,618, runt= 12651msec
    clat (usec): min=433, max=4,702, avg=2466.80, stdev=261.75
    bw (KB/s) : min=399360, max=468992, per=25.04%, avg=415016.36, stdev=5641.57
  cpu          : usr=0.16%, sys=2.76%, ctx=20563, majf=0, minf=88
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/20480, short=0/0
     lat (usec): 500=0.01%, 750=0.01%, 1000=0.10%
     lat (msec): 2=19.38%, 4=80.43%, 10=0.06%

Run status group 0 (all jobs):
  WRITE: io=20,480MB, aggrb=1,619MB/s, minb=1,658MB/s, maxb=1,658MB/s, mint=12651msec, maxt=12651msec

Disk stats (read/write):
  sdb: ios=0/161147, merge=0/0, ticks=0/346123, in_queue=346194, util=99.50%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/1,386M /s] [0/1K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25381
  write: io=20,480MB, bw=1,372MB/s, iops=1,372, runt= 14924msec
    clat (usec): min=666, max=16,821, avg=2911.68, stdev=864.83
    bw (KB/s) : min=333824, max=460800, per=25.04%, avg=351875.79, stdev=11484.03
  cpu          : usr=0.10%, sys=2.16%, ctx=20566, majf=0, minf=88
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/20480, short=0/0
     lat (usec): 750=0.01%, 1000=0.14%
     lat (msec): 2=15.65%, 4=78.63%, 10=3.26%, 20=2.31%

Run status group 0 (all jobs):
  WRITE: io=20,480MB, aggrb=1,372MB/s, minb=1,405MB/s, maxb=1,405MB/s, mint=14924msec, maxt=14924msec

Disk stats (read/write):
  sdb: ios=0/161490, merge=0/0, ticks=0/411564, in_queue=411620, util=99.57%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/1,656M /s] [0/2K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25387
  write: io=96,976MB, bw=1,616MB/s, iops=1,616, runt= 60002msec
    clat (usec): min=476, max=5,212, avg=2471.99, stdev=255.18
    bw (KB/s) : min=395264, max=470099, per=25.02%, avg=414131.86, stdev=3362.06
  cpu          : usr=0.10%, sys=2.49%, ctx=97318, majf=0, minf=92
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/96976, short=0/0
     lat (usec): 500=0.01%, 750=0.01%, 1000=0.02%
     lat (msec): 2=18.67%, 4=81.28%, 10=0.03%

Run status group 0 (all jobs):
  WRITE: io=96,976MB, aggrb=1,616MB/s, minb=1,655MB/s, maxb=1,655MB/s, mint=60002msec, maxt=60002msec

Disk stats (read/write):
  sdb: ios=0/774604, merge=0/0, ticks=0/1662850, in_queue=1662946, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/202M /s] [0/197 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25426
  write: io=11,836MB, bw=197MB/s, iops=197, runt= 60037msec
    clat (usec): min=505, max=293K, avg=20348.19, stdev=15687.08
    bw (KB/s) : min=27330, max=129774, per=25.00%, avg=50470.29, stdev=5092.48
  cpu          : usr=0.03%, sys=0.36%, ctx=11896, majf=0, minf=124
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/11836, short=0/0
     lat (usec): 750=0.03%, 1000=0.06%
     lat (msec): 2=0.53%, 4=3.22%, 10=73.68%, 20=0.30%, 50=3.43%
     lat (msec): 100=16.58%, 250=2.11%, 500=0.04%

Run status group 0 (all jobs):
  WRITE: io=11,836MB, aggrb=197MB/s, minb=202MB/s, maxb=202MB/s, mint=60037msec, maxt=60037msec

Disk stats (read/write):
  sdb: ios=0/94454, merge=0/0, ticks=0/1674944, in_queue=1676060, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=1794597978112 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/144M /s] [0/140 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25434
  write: io=8,502MB, bw=142MB/s, iops=141, runt= 60056msec
    clat (usec): min=446, max=543K, avg=28195.25, stdev=25217.19
    bw (KB/s) : min=11976, max=114413, per=25.37%, avg=36784.37, stdev=5244.56
  cpu          : usr=0.02%, sys=0.26%, ctx=8540, majf=0, minf=88
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/8502, short=0/0
     lat (usec): 500=0.02%, 750=0.01%, 1000=0.08%
     lat (msec): 2=0.78%, 4=6.16%, 10=70.67%, 20=0.13%, 50=1.81%
     lat (msec): 100=8.82%, 250=10.77%, 500=0.72%, 750=0.02%

Run status group 0 (all jobs):
  WRITE: io=8,502MB, aggrb=142MB/s, minb=145MB/s, maxb=145MB/s, mint=60056msec, maxt=60056msec

Disk stats (read/write):
  sdb: ios=0/67857, merge=0/0, ticks=0/1657239, in_queue=1659674, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=1799966687232 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/115M /s] [0/112 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=25442
  write: io=6,721MB, bw=112MB/s, iops=111, runt= 60010msec
    clat (usec): min=494, max=462K, avg=35742.19, stdev=29478.20
    bw (KB/s) : min=15201, max=78113, per=24.98%, avg=28645.25, stdev=2924.34
  cpu          : usr=0.01%, sys=0.22%, ctx=6751, majf=0, minf=891
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/6721, short=0/0
     lat (usec): 500=0.01%, 750=0.03%, 1000=0.10%
     lat (msec): 2=0.83%, 4=5.07%, 10=69.78%, 20=0.04%, 50=0.54%
     lat (msec): 100=5.49%, 250=17.94%, 500=0.15%

Run status group 0 (all jobs):
  WRITE: io=6,721MB, aggrb=112MB/s, minb=115MB/s, maxb=115MB/s, mint=60010msec, maxt=60010msec

Disk stats (read/write):
  sdb: ios=0/53743, merge=0/0, ticks=0/1674622, in_queue=1676045, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
[root@iotest ~]#