[root@iotest ~]# cat fiotester2.sh 
#!/bin/bash

FIO="/usr/bin/fio"
FN="/dev/sdd"
OFFSET="494739152896"
# 5GB are 5.368.709.120 Bytes
DEVSIZE="500107862016"
DEVSIZE10P="50010786201"
BSIOPS="512"
BSTHR="1m"

hdparm -W0 /dev/sdd
hdparm -I /dev/sdd

for TESTCASE in "read" "write"
do 
	for NJ in 1 4
	do
		echo "#################################################################################"
		echo "Executing $TESTCASE tests with numjobs=$NJ"
		for testrun in `seq 1 10`
		do
			case $testrun in
				1)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				2)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				3)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				4)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				5)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				6)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				7)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				8)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				9)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				10)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
			esac
			echo "---------------------------------------------------------------------------------"
			echo "Executing test $testrun: $TESTCMD"
			$TESTCMD
			echo "Sleeping for 30 seconds... ------------------------------------------------------"
			sleep 30
		done
	done
done
[root@iotest ~]# ./fiotester2.sh 

/dev/sdd:
 setting drive write-caching to 0 (off)

/dev/sdd:

ATA device, with non-removable media
	Model Number:       WDC WD5002ABYS-01B1B0                   
	Serial Number:      WD-WCASY0752420
	Firmware Revision:  02.03B02
Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5
Standards:
	Supported: 8 7 6 5 
	Likely used: 8
Configuration:
	Logical		max	current
	cylinders	16383	16383
	heads		16	16
	sectors/track	63	63
	--
	CHS current addressable sectors:   16514064
	LBA    user addressable sectors:  268435455
	LBA48  user addressable sectors:  976773168
	device size with M = 1024*1024:      476940 MBytes
	device size with M = 1000*1000:      500107 MBytes (500 GB)
Capabilities:
	LBA, IORDY(can be disabled)
	Queue depth: 32
	Standby timer values: spec'd by Standard, with device specific minimum
	R/W multiple sector transfer: Max = 16	Current = 16
	Recommended acoustic management value: 128, current value: 254
	DMA: mdma0 mdma1 mdma2 udma0 udma1 udma2 udma3 udma4 udma5 *udma6 
	     Cycle time: min=120ns recommended=120ns
	PIO: pio0 pio1 pio2 pio3 pio4 
	     Cycle time: no flow control=120ns  IORDY flow control=120ns
Commands/features:
	Enabled	Supported:
	   *	SMART feature set
	    	Security Mode feature set
	   *	Power Management feature set
	    	Write cache
	   *	Look-ahead
	   *	Host Protected Area feature set
	   *	WRITE_BUFFER command
	   *	READ_BUFFER command
	   *	NOP cmd
	   *	DOWNLOAD_MICROCODE
	    	Power-Up In Standby feature set
	   *	SET_FEATURES required to spinup after power up
	    	SET_MAX security extension
	   *	Automatic Acoustic Management feature set
	   *	48-bit Address feature set
	   *	Device Configuration Overlay feature set
	   *	Mandatory FLUSH_CACHE
	   *	FLUSH_CACHE_EXT
	   *	SMART error logging
	   *	SMART self-test
	   *	General Purpose Logging feature set
	   *	WRITE_{DMA|MULTIPLE}_FUA_EXT
	   *	64-bit World wide name
	   *	WRITE_UNCORRECTABLE command
	   *	{READ,WRITE}_DMA_EXT_GPL commands
	   *	Segmented DOWNLOAD_MICROCODE
	   *	SATA-I signaling speed (1.5Gb/s)
	   *	SATA-II signaling speed (3.0Gb/s)
	   *	Native Command Queueing (NCQ)
	   *	Phy event counters
	    	DMA Setup Auto-Activate optimization
	   *	Software settings preservation
Security: 
	Master password revision code = 65534
		supported
	not	enabled
	not	locked
		frozen
	not	expired: security count
		supported: enhanced erase
	96min for SECURITY ERASE UNIT. 96min for ENHANCED SECURITY ERASE UNIT.
Checksum: correct
#################################################################################
Executing read tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [5,407K/0K /s] [11K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9188
  read : io=306MB, bw=5,222KB/s, iops=10,443, runt= 60001msec
    clat (usec): min=71, max=19,414, avg=93.85, stdev=31.22
    bw (KB/s) : min= 3870, max= 5314, per=100.02%, avg=5221.85, stdev=185.36
  cpu          : usr=2.25%, sys=60.70%, ctx=626612, majf=0, minf=27
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=626599/0, short=0/0
     lat (usec): 100=84.47%, 250=15.43%, 500=0.05%, 750=0.05%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=306MB, aggrb=5,221KB/s, minb=5,346KB/s, maxb=5,346KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=625693/0, merge=0/0, ticks=55115/0, in_queue=55196, util=92.26%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [5,427K/0K /s] [11K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9193
  read : io=304MB, bw=5,190KB/s, iops=10,380, runt= 60001msec
    clat (usec): min=72, max=20,458, avg=94.44, stdev=34.68
    bw (KB/s) : min= 3203, max= 5314, per=100.00%, avg=5189.97, stdev=335.62
  cpu          : usr=2.26%, sys=60.91%, ctx=622843, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=622833/0, short=0/0
     lat (usec): 100=83.96%, 250=15.95%, 500=0.05%, 750=0.05%, 1000=0.01%
     lat (msec): 2=0.01%, 10=0.01%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=304MB, aggrb=5,190KB/s, minb=5,314KB/s, maxb=5,314KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=621853/0, merge=0/0, ticks=55509/0, in_queue=55569, util=92.85%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [42K/0K /s] [83/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9198
  read : io=2,486KB, bw=42,415B/s, iops=82, runt= 60005msec
    clat (msec): min=2, max=65, avg=12.06, stdev= 3.66
    bw (KB/s) : min=   36, max=   46, per=99.75%, avg=40.90, stdev= 2.00
  cpu          : usr=0.08%, sys=0.60%, ctx=4976, majf=0, minf=136
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=4971/0, short=0/0

     lat (msec): 4=0.84%, 10=28.49%, 20=69.42%, 50=1.23%, 100=0.02%

Run status group 0 (all jobs):
   READ: io=2,485KB, aggrb=41KB/s, minb=42KB/s, maxb=42KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdd: ios=4955/0, merge=0/0, ticks=59739/0, in_queue=59747, util=99.84%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [63K/0K /s] [124/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9205
  read : io=3,902KB, bw=66,589B/s, iops=130, runt= 60004msec
    clat (msec): min=1, max=19, avg= 7.68, stdev= 2.56
    bw (KB/s) : min=   59, max=   71, per=99.35%, avg=64.58, stdev= 2.60
  cpu          : usr=0.14%, sys=0.91%, ctx=7812, majf=0, minf=1372
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7804/0, short=0/0

     lat (msec): 2=0.13%, 4=7.44%, 10=69.66%, 20=22.77%

Run status group 0 (all jobs):
   READ: io=3,902KB, aggrb=65KB/s, minb=66KB/s, maxb=66KB/s, mint=60004msec, maxt=60004msec

Disk stats (read/write):
  sdd: ios=7780/0, merge=0/0, ticks=59723/0, in_queue=59723, util=99.79%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [113M/0K /s] [110/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9239
  read : io=5,120MB, bw=109MB/s, iops=108, runt= 47020msec
    clat (msec): min=7, max=92, avg= 9.18, stdev= 2.34
    bw (KB/s) : min=96755, max=116502, per=100.10%, avg=111619.24, stdev=3935.34
  cpu          : usr=0.04%, sys=1.73%, ctx=5149, majf=0, minf=282
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0

     lat (msec): 10=96.25%, 20=3.24%, 50=0.47%, 100=0.04%

Run status group 0 (all jobs):
   READ: io=5,120MB, aggrb=109MB/s, minb=112MB/s, maxb=112MB/s, mint=47020msec, maxt=47020msec

Disk stats (read/write):
  sdd: ios=10204/0, merge=0/0, ticks=68949/0, in_queue=68948, util=99.31%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [61M/0K /s] [59/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9244
  read : io=3,614MB, bw=61,651KB/s, iops=60, runt= 60027msec
    clat (msec): min=14, max=57, avg=16.61, stdev= 3.16
    bw (KB/s) : min=53685, max=64758, per=100.11%, avg=61717.58, stdev=2463.19
  cpu          : usr=0.02%, sys=0.97%, ctx=3636, majf=0, minf=284
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3614/0, short=0/0

     lat (msec): 20=94.08%, 50=5.87%, 100=0.06%

Run status group 0 (all jobs):
   READ: io=3,614MB, aggrb=61,651KB/s, minb=63,130KB/s, maxb=63,130KB/s, mint=60027msec, maxt=60027msec

Disk stats (read/write):
  sdd: ios=7217/0, merge=0/0, ticks=88856/0, in_queue=88868, util=99.69%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [114M/0K /s] [111/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9249
  read : io=6,542MB, bw=109MB/s, iops=109, runt= 60007msec
    clat (msec): min=7, max=83, avg= 9.17, stdev= 2.23
    bw (KB/s) : min=88445, max=118546, per=100.16%, avg=111814.92, stdev=4449.22
  cpu          : usr=0.05%, sys=1.76%, ctx=6596, majf=0, minf=281
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=6542/0, short=0/0

     lat (msec): 10=96.41%, 20=3.09%, 50=0.46%, 100=0.05%

Run status group 0 (all jobs):
   READ: io=6,542MB, aggrb=109MB/s, minb=112MB/s, maxb=112MB/s, mint=60007msec, maxt=60007msec

Disk stats (read/write):
  sdd: ios=13063/0, merge=0/0, ticks=88108/0, in_queue=88107, util=99.35%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [64M/0K /s] [62/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9254
  read : io=3,642MB, bw=62,143KB/s, iops=60, runt= 60013msec
    clat (msec): min=8, max=85, avg=16.47, stdev= 3.75
    bw (KB/s) : min=53789, max=69323, per=100.09%, avg=62198.74, stdev=2885.70
  cpu          : usr=0.03%, sys=0.97%, ctx=3669, majf=0, minf=289
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3642/0, short=0/0

     lat (msec): 10=0.16%, 20=90.06%, 50=9.69%, 100=0.08%

Run status group 0 (all jobs):
   READ: io=3,642MB, aggrb=62,143KB/s, minb=63,634KB/s, maxb=63,634KB/s, mint=60013msec, maxt=60013msec

Disk stats (read/write):
  sdd: ios=7271/0, merge=0/0, ticks=102054/0, in_queue=102063, util=99.65%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [42M/0K /s] [40/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9259
  read : io=2,475MB, bw=42,226KB/s, iops=41, runt= 60020msec
    clat (msec): min=17, max=78, avg=24.24, stdev= 4.42
    bw (KB/s) : min=37343, max=45821, per=100.08%, avg=42259.41, stdev=1523.87
  cpu          : usr=0.03%, sys=0.66%, ctx=2492, majf=0, minf=281
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2475/0, short=0/0

     lat (msec): 20=10.71%, 50=89.01%, 100=0.28%

Run status group 0 (all jobs):
   READ: io=2,475MB, aggrb=42,225KB/s, minb=43,239KB/s, maxb=43,239KB/s, mint=60020msec, maxt=60020msec

Disk stats (read/write):
  sdd: ios=4940/0, merge=0/0, ticks=98575/0, in_queue=98590, util=99.82%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [42M/0K /s] [40/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9266
  read : io=2,540MB, bw=43,341KB/s, iops=42, runt= 60012msec
    clat (msec): min=12, max=56, avg=23.62, stdev= 4.85
    bw (KB/s) : min=37705, max=48761, per=100.12%, avg=43389.91, stdev=2097.10
  cpu          : usr=0.03%, sys=0.67%, ctx=2555, majf=0, minf=359
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2540/0, short=0/0

     lat (msec): 20=22.60%, 50=77.20%, 100=0.20%

Run status group 0 (all jobs):
   READ: io=2,540MB, aggrb=43,340KB/s, minb=44,380KB/s, maxb=44,380KB/s, mint=60012msec, maxt=60012msec

Disk stats (read/write):
  sdd: ios=5070/0, merge=0/0, ticks=104855/0, in_queue=104875, util=99.78%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing read tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [3,765K/0K /s] [7K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9272
  read : io=214MB, bw=3,658KB/s, iops=7,316, runt= 60001msec
    clat (usec): min=198, max=28,540, avg=544.58, stdev=72.97
    bw (KB/s) : min=  876, max= 1005, per=25.02%, avg=915.19, stdev= 7.42
  cpu          : usr=0.42%, sys=0.78%, ctx=439174, majf=0, minf=104
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=438977/0, short=0/0
     lat (usec): 250=0.01%, 500=13.48%, 750=86.32%, 1000=0.15%
     lat (msec): 2=0.02%, 4=0.01%, 10=0.02%, 20=0.01%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=214MB, aggrb=3,658KB/s, minb=3,745KB/s, maxb=3,745KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=437445/0, merge=830/0, ticks=210341/0, in_queue=210452, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [3,735K/0K /s] [7K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9280
  read : io=215MB, bw=3,675KB/s, iops=7,350, runt= 60001msec
    clat (usec): min=209, max=26,812, avg=541.99, stdev=67.36
    bw (KB/s) : min=  877, max= 1058, per=25.02%, avg=919.45, stdev=11.25
  cpu          : usr=0.41%, sys=0.83%, ctx=441244, majf=0, minf=116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=441057/0, short=0/0
     lat (usec): 250=0.01%, 500=21.28%, 750=78.38%, 1000=0.29%
     lat (msec): 2=0.03%, 4=0.01%, 10=0.02%, 20=0.01%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=215MB, aggrb=3,675KB/s, minb=3,763KB/s, maxb=3,763KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=437568/0, merge=2815/0, ticks=209283/0, in_queue=209409, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [0.0% done] [42K/0K /s] [83/0 iops] [eta 557d:11h:50m:11s]
file1: (groupid=0, jobs=4): err= 0: pid=9288
  read : io=2,489KB, bw=42,407B/s, iops=82, runt= 60101msec
    clat (msec): min=2, max=438, avg=48.41, stdev=50.82
    bw (KB/s) : min=    4, max=   17, per=25.64%, avg=10.51, stdev= 1.91
  cpu          : usr=0.05%, sys=0.13%, ctx=4984, majf=0, minf=150
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=4978/0, short=0/0

     lat (msec): 4=0.62%, 10=24.93%, 20=61.95%, 50=1.08%, 100=0.06%
     lat (msec): 250=0.24%, 500=11.11%

Run status group 0 (all jobs):
   READ: io=2,489KB, aggrb=41KB/s, minb=42KB/s, maxb=42KB/s, mint=60101msec, maxt=60101msec

Disk stats (read/write):
  sdd: ios=4957/0, merge=0/0, ticks=238914/0, in_queue=239547, util=99.91%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [0.0% done] [65K/0K /s] [128/0 iops] [eta 34d:17h:21m:38s]
file1: (groupid=0, jobs=4): err= 0: pid=9296
  read : io=3,929KB, bw=66,882B/s, iops=130, runt= 60155msec
    clat (msec): min=1, max=431, avg=30.40, stdev=40.84
    bw (KB/s) : min=    7, max=   29, per=26.46%, avg=17.20, stdev= 3.43
  cpu          : usr=0.07%, sys=0.21%, ctx=7875, majf=0, minf=2319
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7858/0, short=0/0

     lat (msec): 2=0.14%, 4=6.96%, 10=64.98%, 20=20.58%, 50=0.04%
     lat (msec): 250=0.10%, 500=7.20%

Run status group 0 (all jobs):
   READ: io=3,929KB, aggrb=65KB/s, minb=66KB/s, maxb=66KB/s, mint=60155msec, maxt=60155msec

Disk stats (read/write):
  sdd: ios=7835/0, merge=0/0, ticks=239100/0, in_queue=239547, util=99.92%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [129M/0K /s] [125/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9335
  read : io=7,360MB, bw=123MB/s, iops=122, runt= 60037msec
    clat (msec): min=4, max=284, avg=46.44, stdev= 4.40
    bw (KB/s) : min= 6440, max=44433, per=27.21%, avg=34154.42, stdev=833.50
  cpu          : usr=0.02%, sys=0.28%, ctx=7394, majf=0, minf=1132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7360/0, short=0/0

     lat (msec): 10=0.16%, 20=15.01%, 50=78.78%, 100=0.69%, 250=5.34%
     lat (msec): 500=0.01%

Run status group 0 (all jobs):
   READ: io=7,360MB, aggrb=123MB/s, minb=126MB/s, maxb=126MB/s, mint=60037msec, maxt=60037msec

Disk stats (read/write):
  sdd: ios=14692/0, merge=0/0, ticks=453659/0, in_queue=453813, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [124M/0K /s] [121/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9348
  read : io=6,951MB, bw=116MB/s, iops=115, runt= 60013msec
    clat (msec): min=6, max=198, avg=34.53, stdev=18.50
    bw (KB/s) : min=14054, max=44846, per=25.11%, avg=29783.17, stdev=3180.48
  cpu          : usr=0.02%, sys=0.28%, ctx=6979, majf=0, minf=1136
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=6951/0, short=0/0

     lat (msec): 10=31.72%, 20=14.27%, 50=36.28%, 100=4.16%, 250=13.57%

Run status group 0 (all jobs):
   READ: io=6,951MB, aggrb=116MB/s, minb=119MB/s, maxb=119MB/s, mint=60013msec, maxt=60013msec

Disk stats (read/write):
  sdd: ios=13876/0, merge=0/0, ticks=448165/0, in_queue=448410, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [125M/0K /s] [122/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9356
  read : io=7,417MB, bw=124MB/s, iops=123, runt= 60042msec
    clat (msec): min=4, max=280, avg=46.34, stdev= 4.72
    bw (KB/s) : min= 6340, max=44860, per=27.23%, avg=34444.67, stdev=777.03
  cpu          : usr=0.02%, sys=0.30%, ctx=7443, majf=0, minf=1128
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7417/0, short=0/0

     lat (msec): 10=0.23%, 20=15.88%, 50=78.13%, 100=0.46%, 250=5.29%
     lat (msec): 500=0.01%

Run status group 0 (all jobs):
   READ: io=7,417MB, aggrb=124MB/s, minb=126MB/s, maxb=126MB/s, mint=60042msec, maxt=60042msec

Disk stats (read/write):
  sdd: ios=14805/0, merge=0/0, ticks=453610/0, in_queue=453811, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [65M/0K /s] [63/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9364
  read : io=3,647MB, bw=62,173KB/s, iops=60, runt= 60067msec
    clat (msec): min=11, max=356, avg=59.93, stdev=12.40
    bw (KB/s) : min= 6440, max=42751, per=29.45%, avg=18312.00, stdev=1326.22
  cpu          : usr=0.01%, sys=0.13%, ctx=3662, majf=0, minf=1148
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3647/0, short=0/0

     lat (msec): 20=3.65%, 50=38.61%, 100=43.65%, 250=13.68%, 500=0.41%

Run status group 0 (all jobs):
   READ: io=3,647MB, aggrb=62,172KB/s, minb=63,664KB/s, maxb=63,664KB/s, mint=60067msec, maxt=60067msec

Disk stats (read/write):
  sdd: ios=7274/0, merge=0/0, ticks=456297/0, in_queue=456531, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [42M/0K /s] [40/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9372
  read : io=2,471MB, bw=42,111KB/s, iops=41, runt= 60086msec
    clat (msec): min=20, max=245, avg=99.10, stdev= 8.36
    bw (KB/s) : min= 6481, max=14760, per=24.63%, avg=10370.55, stdev=583.54
  cpu          : usr=0.01%, sys=0.09%, ctx=2483, majf=0, minf=1132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2471/0, short=0/0

     lat (msec): 50=0.36%, 100=69.41%, 250=30.23%

Run status group 0 (all jobs):
   READ: io=2,471MB, aggrb=42,111KB/s, minb=43,122KB/s, maxb=43,122KB/s, mint=60086msec, maxt=60086msec

Disk stats (read/write):
  sdd: ios=4927/0, merge=0/0, ticks=457359/0, in_queue=457804, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [44M/0K /s] [42/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9382
  read : io=2,559MB, bw=43,626KB/s, iops=42, runt= 60066msec
    clat (msec): min=14, max=366, avg=124.98, stdev=10.01
    bw (KB/s) : min= 3506, max=28276, per=20.25%, avg=8834.03, stdev=522.77
  cpu          : usr=0.01%, sys=0.09%, ctx=2568, majf=0, minf=1417
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2559/0, short=0/0

     lat (msec): 20=0.43%, 50=1.41%, 100=64.52%, 250=33.33%, 500=0.31%

Run status group 0 (all jobs):
   READ: io=2,559MB, aggrb=43,625KB/s, minb=44,672KB/s, maxb=44,672KB/s, mint=60066msec, maxt=60066msec

Disk stats (read/write):
  sdd: ios=5104/0, merge=0/0, ticks=461983/0, in_queue=462286, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/61K /s] [0/119 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9391
  write: io=3,587KB, bw=61,205B/s, iops=119, runt= 60004msec
    clat (usec): min=1,233, max=16,763, avg=8360.97, stdev=533.36
    bw (KB/s) : min=   58, max=   60, per=101.32%, avg=59.78, stdev= 0.43
  cpu          : usr=0.09%, sys=0.80%, ctx=7180, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/7173, short=0/0

     lat (msec): 2=0.04%, 10=99.58%, 20=0.38%

Run status group 0 (all jobs):
  WRITE: io=3,586KB, aggrb=59KB/s, minb=61KB/s, maxb=61KB/s, mint=60004msec, maxt=60004msec

Disk stats (read/write):
  sdd: ios=0/7160, merge=0/0, ticks=0/59828, in_queue=59834, util=99.99%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/61K /s] [0/120 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9396
  write: io=3,600KB, bw=61,430B/s, iops=119, runt= 60001msec
    clat (usec): min=1,196, max=9,567, avg=8330.35, stdev=148.54
    bw (KB/s) : min=   59, max=   60, per=101.68%, avg=59.99, stdev= 0.09
  cpu          : usr=0.07%, sys=0.83%, ctx=7204, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/7199, short=0/0

     lat (msec): 2=0.04%, 10=99.96%

Run status group 0 (all jobs):
  WRITE: io=3,599KB, aggrb=59KB/s, minb=61KB/s, maxb=61KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=0/7188, merge=0/0, ticks=0/59854, in_queue=59857, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/38K /s] [0/75 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9430
  write: io=2,320KB, bw=39,583B/s, iops=77, runt= 60017msec
    clat (msec): min=2, max=22, avg=12.92, stdev= 3.55
    bw (KB/s) : min=   34, max=   42, per=100.31%, avg=38.12, stdev= 1.80
  cpu          : usr=0.08%, sys=0.48%, ctx=4644, majf=0, minf=101
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/4640, short=0/0

     lat (msec): 4=0.17%, 10=21.57%, 20=75.69%, 50=2.56%

Run status group 0 (all jobs):
  WRITE: io=2,320KB, aggrb=38KB/s, minb=39KB/s, maxb=39KB/s, mint=60017msec, maxt=60017msec

Disk stats (read/write):
  sdd: ios=0/4626, merge=0/0, ticks=0/59761, in_queue=59764, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/58K /s] [0/114 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9435
  write: io=3,420KB, bw=58,367B/s, iops=113, runt= 60001msec
    clat (msec): min=1, max=19, avg= 8.76, stdev= 2.55
    bw (KB/s) : min=   51, max=   61, per=101.06%, avg=56.59, stdev= 2.37
  cpu          : usr=0.13%, sys=0.70%, ctx=6851, majf=0, minf=1948
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/6840, short=0/0

     lat (msec): 2=0.01%, 4=1.42%, 10=62.75%, 20=35.82%

Run status group 0 (all jobs):
  WRITE: io=3,420KB, aggrb=56KB/s, minb=58KB/s, maxb=58KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=0/6820, merge=0/0, ticks=0/59738, in_queue=59742, util=99.85%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/41M /s] [0/39 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9442
  write: io=2,335MB, bw=39,841KB/s, iops=38, runt= 60015msec
    clat (msec): min=17, max=42, avg=25.70, stdev= 1.47
    bw (KB/s) : min=38374, max=40878, per=100.12%, avg=39888.91, stdev=514.79
  cpu          : usr=0.03%, sys=1.02%, ctx=2365, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2335, short=0/0

     lat (msec): 20=0.39%, 50=99.61%

Run status group 0 (all jobs):
  WRITE: io=2,335MB, aggrb=39,840KB/s, minb=40,796KB/s, maxb=40,796KB/s, mint=60015msec, maxt=60015msec

Disk stats (read/write):
  sdd: ios=0/4661, merge=0/0, ticks=0/89109, in_queue=89130, util=99.91%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/29M /s] [0/27 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9447
  write: io=1,764MB, bw=30,098KB/s, iops=29, runt= 60016msec
    clat (msec): min=31, max=66, avg=34.02, stdev= 3.97
    bw (KB/s) : min=26050, max=31690, per=100.10%, avg=30126.67, stdev=1076.81
  cpu          : usr=0.00%, sys=0.71%, ctx=1793, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1764, short=0/0

     lat (msec): 50=99.55%, 100=0.45%

Run status group 0 (all jobs):
  WRITE: io=1,764MB, aggrb=30,097KB/s, minb=30,819KB/s, maxb=30,819KB/s, mint=60016msec, maxt=60016msec

Disk stats (read/write):
  sdd: ios=0/3521, merge=0/0, ticks=0/89234, in_queue=89260, util=99.95%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/40M /s] [0/38 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9452
  write: io=2,335MB, bw=39,844KB/s, iops=38, runt= 60010msec
    clat (msec): min=17, max=42, avg=25.70, stdev= 1.48
    bw (KB/s) : min=37778, max=40878, per=100.12%, avg=39892.02, stdev=575.83
  cpu          : usr=0.03%, sys=0.99%, ctx=2357, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2335, short=0/0

     lat (msec): 20=0.34%, 50=99.66%

Run status group 0 (all jobs):
  WRITE: io=2,335MB, aggrb=39,844KB/s, minb=40,800KB/s, maxb=40,800KB/s, mint=60010msec, maxt=60010msec

Disk stats (read/write):
  sdd: ios=0/4662, merge=0/0, ticks=0/89005, in_queue=89014, util=99.90%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/41M /s] [0/39 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9457
  write: io=2,398MB, bw=40,917KB/s, iops=39, runt= 60013msec
    clat (msec): min=18, max=36, avg=25.02, stdev= 2.62
    bw (KB/s) : min=39009, max=43239, per=100.08%, avg=40950.15, stdev=902.02
  cpu          : usr=0.02%, sys=1.03%, ctx=2429, majf=0, minf=28
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2398, short=0/0

     lat (msec): 20=1.21%, 50=98.79%

Run status group 0 (all jobs):
  WRITE: io=2,398MB, aggrb=40,917KB/s, minb=41,899KB/s, maxb=41,899KB/s, mint=60013msec, maxt=60013msec

Disk stats (read/write):
  sdd: ios=0/4787, merge=0/0, ticks=0/88383, in_queue=88398, util=99.87%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/32M /s] [0/30 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9462
  write: io=1,828MB, bw=31,182KB/s, iops=30, runt= 60031msec
    clat (msec): min=26, max=59, avg=32.83, stdev= 3.51
    bw (KB/s) : min=28818, max=32768, per=100.10%, avg=31211.95, stdev=869.44
  cpu          : usr=0.01%, sys=0.78%, ctx=1842, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1828, short=0/0

     lat (msec): 50=99.78%, 100=0.22%

Run status group 0 (all jobs):
  WRITE: io=1,828MB, aggrb=31,181KB/s, minb=31,930KB/s, maxb=31,930KB/s, mint=60031msec, maxt=60031msec

Disk stats (read/write):
  sdd: ios=0/3648, merge=0/0, ticks=0/88760, in_queue=88776, util=99.95%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/31M /s] [0/29 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9469
  write: io=1,825MB, bw=31,146KB/s, iops=30, runt= 60002msec
    clat (msec): min=21, max=56, avg=32.87, stdev= 4.83
    bw (KB/s) : min=28656, max=34608, per=100.12%, avg=31181.29, stdev=1235.09
  cpu          : usr=0.02%, sys=0.83%, ctx=1854, majf=0, minf=85
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1825, short=0/0

     lat (msec): 50=99.45%, 100=0.55%

Run status group 0 (all jobs):
  WRITE: io=1,825MB, aggrb=31,145KB/s, minb=31,893KB/s, maxb=31,893KB/s, mint=60002msec, maxt=60002msec

Disk stats (read/write):
  sdd: ios=0/3645, merge=0/0, ticks=0/93630, in_queue=93648, util=99.93%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/122K /s] [0/239 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9475
  write: io=7,076KB, bw=118KB/s, iops=235, runt= 60012msec
    clat (msec): min=8, max=108, avg=16.96, stdev= 2.30
    bw (KB/s) : min=   24, max=   30, per=25.17%, avg=29.45, stdev= 0.74
  cpu          : usr=0.03%, sys=0.07%, ctx=14165, majf=0, minf=100
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/14152, short=0/0

     lat (msec): 10=0.01%, 20=99.29%, 50=0.39%, 100=0.16%, 250=0.15%

Run status group 0 (all jobs):
  WRITE: io=7,076KB, aggrb=117KB/s, minb=120KB/s, maxb=120KB/s, mint=60012msec, maxt=60012msec

Disk stats (read/write):
  sdd: ios=0/7078, merge=0/7050, ticks=0/119964, in_queue=119987, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/122K /s] [0/238 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9483
  write: io=7,070KB, bw=118KB/s, iops=235, runt= 60010msec
    clat (msec): min=10, max=100, avg=16.98, stdev= 2.22
    bw (KB/s) : min=   25, max=   30, per=25.15%, avg=29.42, stdev= 0.76
  cpu          : usr=0.04%, sys=0.08%, ctx=14153, majf=0, minf=100
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/14140, short=0/0

     lat (msec): 20=99.08%, 50=0.61%, 100=0.24%, 250=0.07%

Run status group 0 (all jobs):
  WRITE: io=7,070KB, aggrb=117KB/s, minb=120KB/s, maxb=120KB/s, mint=60010msec, maxt=60010msec

Disk stats (read/write):
  sdd: ios=0/7083, merge=0/7032, ticks=0/120291, in_queue=120296, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 3 (f=3): [_www] [50.4% done] [0K/39K /s] [0/77 iops] [eta 01m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9520
  write: io=2,315KB, bw=39,489B/s, iops=77, runt= 60030msec
    clat (msec): min=6, max=161, avg=51.87, stdev= 4.02
    bw (KB/s) : min=    7, max=   14, per=23.99%, avg= 9.12, stdev= 0.30
  cpu          : usr=0.04%, sys=0.03%, ctx=4640, majf=0, minf=165
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/4630, short=0/0

     lat (msec): 10=0.02%, 20=0.26%, 50=40.09%, 100=59.55%, 250=0.09%

Run status group 0 (all jobs):
  WRITE: io=2,315KB, aggrb=38KB/s, minb=39KB/s, maxb=39KB/s, mint=60030msec, maxt=60030msec

Disk stats (read/write):
  sdd: ios=0/4617, merge=0/0, ticks=0/239330, in_queue=239375, util=99.94%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 3 (f=3): [ww_w] [50.4% done] [0K/58K /s] [0/114 iops] [eta 01m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9531
  write: io=3,477KB, bw=59,304B/s, iops=115, runt= 60028msec
    clat (msec): min=4, max=50, avg=34.47, stdev= 2.68
    bw (KB/s) : min=   13, max=   23, per=24.41%, avg=13.92, stdev= 0.34
  cpu          : usr=0.05%, sys=0.04%, ctx=6966, majf=0, minf=2249
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/6953, short=0/0

     lat (msec): 10=0.13%, 20=0.36%, 50=99.50%, 100=0.01%

Run status group 0 (all jobs):
  WRITE: io=3,476KB, aggrb=57KB/s, minb=59KB/s, maxb=59KB/s, mint=60028msec, maxt=60028msec

Disk stats (read/write):
  sdd: ios=0/6934, merge=0/0, ticks=0/239159, in_queue=239195, util=99.96%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/40M /s] [0/38 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9541
  write: io=2,386MB, bw=40,663KB/s, iops=39, runt= 60085msec
    clat (usec): min=24,208, max=109K, avg=100708.83, stdev=751.18
    bw (KB/s) : min= 9961, max=11658, per=25.03%, avg=10175.98, stdev=34.06
  cpu          : usr=0.00%, sys=0.07%, ctx=2401, majf=0, minf=100
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2386, short=0/0

     lat (msec): 50=0.08%, 100=6.79%, 250=93.13%

Run status group 0 (all jobs):
  WRITE: io=2,386MB, aggrb=40,663KB/s, minb=41,639KB/s, maxb=41,639KB/s, mint=60085msec, maxt=60085msec

Disk stats (read/write):
  sdd: ios=0/4757, merge=0/0, ticks=0/447175, in_queue=447538, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/33M /s] [0/31 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9549
  write: io=1,900MB, bw=32,365KB/s, iops=31, runt= 60114msec
    clat (msec): min=52, max=175, avg=126.45, stdev= 4.98
    bw (KB/s) : min= 7288, max= 9208, per=25.01%, avg=8094.41, stdev=102.22
  cpu          : usr=0.00%, sys=0.06%, ctx=1910, majf=0, minf=108
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1900, short=0/0

     lat (msec): 100=0.11%, 250=99.89%

Run status group 0 (all jobs):
  WRITE: io=1,900MB, aggrb=32,365KB/s, minb=33,141KB/s, maxb=33,141KB/s, mint=60114msec, maxt=60114msec

Disk stats (read/write):
  sdd: ios=0/3786, merge=0/0, ticks=0/446687, in_queue=447180, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/41M /s] [0/39 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9557
  write: io=2,382MB, bw=40,595KB/s, iops=39, runt= 60085msec
    clat (msec): min=24, max=125, avg=100.88, stdev= 1.00
    bw (KB/s) : min= 9570, max=11658, per=25.03%, avg=10159.32, stdev=48.75
  cpu          : usr=0.01%, sys=0.07%, ctx=2392, majf=0, minf=100
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2382, short=0/0

     lat (msec): 50=0.08%, 100=6.51%, 250=93.41%

Run status group 0 (all jobs):
  WRITE: io=2,382MB, aggrb=40,595KB/s, minb=41,569KB/s, maxb=41,569KB/s, mint=60085msec, maxt=60085msec

Disk stats (read/write):
  sdd: ios=0/4750, merge=0/0, ticks=0/447423, in_queue=447732, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/41M /s] [0/39 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9565
  write: io=2,419MB, bw=41,222KB/s, iops=40, runt= 60090msec
    clat (msec): min=28, max=126, avg=99.26, stdev= 2.95
    bw (KB/s) : min= 9678, max=11906, per=25.02%, avg=10314.99, stdev=137.40
  cpu          : usr=0.01%, sys=0.07%, ctx=2436, majf=0, minf=116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2419, short=0/0

     lat (msec): 50=0.04%, 100=55.02%, 250=44.94%

Run status group 0 (all jobs):
  WRITE: io=2,419MB, aggrb=41,222KB/s, minb=42,211KB/s, maxb=42,211KB/s, mint=60090msec, maxt=60090msec

Disk stats (read/write):
  sdd: ios=0/4823, merge=0/0, ticks=0/446952, in_queue=447277, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/31M /s] [0/29 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9575
  write: io=1,837MB, bw=31,287KB/s, iops=30, runt= 60123msec
    clat (msec): min=48, max=167, avg=130.78, stdev= 3.50
    bw (KB/s) : min= 7249, max= 9142, per=25.05%, avg=7838.40, stdev=102.17
  cpu          : usr=0.00%, sys=0.04%, ctx=1839, majf=0, minf=112
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1837, short=0/0

     lat (msec): 50=0.05%, 100=0.05%, 250=99.89%

Run status group 0 (all jobs):
  WRITE: io=1,837MB, aggrb=31,287KB/s, minb=32,038KB/s, maxb=32,038KB/s, mint=60123msec, maxt=60123msec

Disk stats (read/write):
  sdd: ios=0/3661, merge=0/0, ticks=0/447560, in_queue=447979, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/33M /s] [0/31 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9583
  write: io=1,836MB, bw=31,268KB/s, iops=30, runt= 60127msec
    clat (msec): min=44, max=172, avg=130.85, stdev= 5.25
    bw (KB/s) : min= 6965, max= 8722, per=25.04%, avg=7830.46, stdev=145.24
  cpu          : usr=0.00%, sys=0.05%, ctx=1844, majf=0, minf=325
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/1836, short=0/0

     lat (msec): 50=0.05%, 100=0.11%, 250=99.84%

Run status group 0 (all jobs):
  WRITE: io=1,836MB, aggrb=31,268KB/s, minb=32,018KB/s, maxb=32,018KB/s, mint=60127msec, maxt=60127msec

Disk stats (read/write):
  sdd: ios=0/3658, merge=0/0, ticks=0/452088, in_queue=452621, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
[root@iotest ~]#