[root@iotest ~]# cat fiotester2.sh 
#!/bin/bash

FIO="/usr/bin/fio"
FN="/dev/sdd"
OFFSET="494739152896"
# 5GB are 5.368.709.120 Bytes
DEVSIZE="500107862016"
DEVSIZE10P="50010786201"
BSIOPS="512"
BSTHR="1m"

hdparm -W1 /dev/sdd
hdparm -I /dev/sdd

for TESTCASE in "read" "write"
do 
	for NJ in 1 4
	do
		echo "#################################################################################"
		echo "Executing $TESTCASE tests with numjobs=$NJ"
		for testrun in `seq 1 10`
		do
			case $testrun in
				1)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				2)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				3)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				4)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				5)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				6)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				7)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				8)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				9)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				10)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
			esac
			echo "---------------------------------------------------------------------------------"
			echo "Executing test $testrun: $TESTCMD"
			$TESTCMD
			echo "Sleeping for 30 seconds... ------------------------------------------------------"
			sleep 30
		done
	done
done
[root@iotest ~]# ./fiotester2.sh 

/dev/sdd:
 setting drive write-caching to 1 (on)

/dev/sdd:

ATA device, with non-removable media
	Model Number:       WDC WD5002ABYS-01B1B0                   
	Serial Number:      WD-WCASY0752420
	Firmware Revision:  02.03B02
Transport: Serial, SATA 1.0a, SATA II Extensions, SATA Rev 2.5
Standards:
	Supported: 8 7 6 5 
	Likely used: 8
Configuration:
	Logical		max	current
	cylinders	16383	16383
	heads		16	16
	sectors/track	63	63
	--
	CHS current addressable sectors:   16514064
	LBA    user addressable sectors:  268435455
	LBA48  user addressable sectors:  976773168
	device size with M = 1024*1024:      476940 MBytes
	device size with M = 1000*1000:      500107 MBytes (500 GB)
Capabilities:
	LBA, IORDY(can be disabled)
	Queue depth: 32
	Standby timer values: spec'd by Standard, with device specific minimum
	R/W multiple sector transfer: Max = 16	Current = 16
	Recommended acoustic management value: 128, current value: 254
	DMA: mdma0 mdma1 mdma2 udma0 udma1 udma2 udma3 udma4 udma5 *udma6 
	     Cycle time: min=120ns recommended=120ns
	PIO: pio0 pio1 pio2 pio3 pio4 
	     Cycle time: no flow control=120ns  IORDY flow control=120ns
Commands/features:
	Enabled	Supported:
	   *	SMART feature set
	    	Security Mode feature set
	   *	Power Management feature set
	   *	Write cache
	   *	Look-ahead
	   *	Host Protected Area feature set
	   *	WRITE_BUFFER command
	   *	READ_BUFFER command
	   *	NOP cmd
	   *	DOWNLOAD_MICROCODE
	    	Power-Up In Standby feature set
	   *	SET_FEATURES required to spinup after power up
	    	SET_MAX security extension
	   *	Automatic Acoustic Management feature set
	   *	48-bit Address feature set
	   *	Device Configuration Overlay feature set
	   *	Mandatory FLUSH_CACHE
	   *	FLUSH_CACHE_EXT
	   *	SMART error logging
	   *	SMART self-test
	   *	General Purpose Logging feature set
	   *	WRITE_{DMA|MULTIPLE}_FUA_EXT
	   *	64-bit World wide name
	   *	WRITE_UNCORRECTABLE command
	   *	{READ,WRITE}_DMA_EXT_GPL commands
	   *	Segmented DOWNLOAD_MICROCODE
	   *	SATA-I signaling speed (1.5Gb/s)
	   *	SATA-II signaling speed (3.0Gb/s)
	   *	Native Command Queueing (NCQ)
	   *	Phy event counters
	    	DMA Setup Auto-Activate optimization
	   *	Software settings preservation
Security: 
	Master password revision code = 65534
		supported
	not	enabled
	not	locked
		frozen
	not	expired: security count
		supported: enhanced erase
	96min for SECURITY ERASE UNIT. 96min for ENHANCED SECURITY ERASE UNIT.
Checksum: correct
#################################################################################
Executing read tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [5,372K/0K /s] [10K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9627
  read : io=306MB, bw=5,230KB/s, iops=10,459, runt= 60001msec
    clat (usec): min=72, max=8,866, avg=93.72, stdev=24.50
    bw (KB/s) : min= 3807, max= 5320, per=100.03%, avg=5230.52, stdev=184.65
  cpu          : usr=2.33%, sys=61.33%, ctx=627602, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=627599/0, short=0/0
     lat (usec): 100=85.07%, 250=14.84%, 500=0.04%, 750=0.05%
     lat (msec): 2=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=306MB, aggrb=5,229KB/s, minb=5,355KB/s, maxb=5,355KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=626683/0, merge=0/0, ticks=55219/0, in_queue=55287, util=92.38%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [5,400K/0K /s] [11K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9632
  read : io=306MB, bw=5,216KB/s, iops=10,431, runt= 60001msec
    clat (usec): min=72, max=11,248, avg=93.96, stdev=26.75
    bw (KB/s) : min= 3267, max= 5316, per=100.01%, avg=5215.56, stdev=248.06
  cpu          : usr=2.16%, sys=61.48%, ctx=625902, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=625882/0, short=0/0
     lat (usec): 100=84.59%, 250=15.32%, 500=0.05%, 750=0.05%
     lat (msec): 2=0.01%, 10=0.01%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=306MB, aggrb=5,215KB/s, minb=5,340KB/s, maxb=5,340KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=624903/0, merge=0/0, ticks=54952/0, in_queue=55036, util=91.94%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [42K/0K /s] [82/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9637
  read : io=2,486KB, bw=42,424B/s, iops=82, runt= 60005msec
    clat (msec): min=2, max=22, avg=12.06, stdev= 3.59
    bw (KB/s) : min=   37, max=   47, per=99.98%, avg=40.99, stdev= 1.86
  cpu          : usr=0.12%, sys=0.53%, ctx=4979, majf=0, minf=164
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=4972/0, short=0/0

     lat (msec): 4=0.86%, 10=28.46%, 20=69.39%, 50=1.29%

Run status group 0 (all jobs):
   READ: io=2,486KB, aggrb=41KB/s, minb=42KB/s, maxb=42KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdd: ios=4955/0, merge=0/0, ticks=59684/0, in_queue=59690, util=99.77%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [63K/0K /s] [124/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9642
  read : io=3,901KB, bw=66,570B/s, iops=130, runt= 60006msec
    clat (msec): min=1, max=14, avg= 7.68, stdev= 2.56
    bw (KB/s) : min=   59, max=   70, per=99.34%, avg=64.57, stdev= 2.57
  cpu          : usr=0.16%, sys=0.80%, ctx=7811, majf=0, minf=1556
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7802/0, short=0/0

     lat (msec): 2=0.12%, 4=7.51%, 10=69.64%, 20=22.74%

Run status group 0 (all jobs):
   READ: io=3,901KB, aggrb=65KB/s, minb=66KB/s, maxb=66KB/s, mint=60006msec, maxt=60006msec

Disk stats (read/write):
  sdd: ios=7777/0, merge=0/0, ticks=59691/0, in_queue=59697, util=99.75%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [112M/0K /s] [109/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9649
  read : io=5,120MB, bw=108MB/s, iops=107, runt= 47448msec
    clat (msec): min=7, max=83, avg= 9.26, stdev= 2.65
    bw (KB/s) : min=89291, max=117375, per=100.09%, avg=110594.13, stdev=5670.53
  cpu          : usr=0.06%, sys=1.71%, ctx=5156, majf=0, minf=281
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0

     lat (msec): 10=95.76%, 20=3.44%, 50=0.72%, 100=0.08%

Run status group 0 (all jobs):
   READ: io=5,120MB, aggrb=108MB/s, minb=110MB/s, maxb=110MB/s, mint=47448msec, maxt=47448msec

Disk stats (read/write):
  sdd: ios=10219/0, merge=0/0, ticks=69649/0, in_queue=69653, util=99.29%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [59M/0K /s] [57/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9654
  read : io=3,605MB, bw=61,522KB/s, iops=60, runt= 60003msec
    clat (msec): min=14, max=73, avg=16.64, stdev= 3.22
    bw (KB/s) : min=50592, max=65274, per=100.10%, avg=61580.53, stdev=2696.33
  cpu          : usr=0.03%, sys=0.96%, ctx=3633, majf=0, minf=285
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3605/0, short=0/0

     lat (msec): 20=93.79%, 50=6.16%, 100=0.06%

Run status group 0 (all jobs):
   READ: io=3,605MB, aggrb=61,522KB/s, minb=62,998KB/s, maxb=62,998KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdd: ios=7198/0, merge=0/0, ticks=88935/0, in_queue=88941, util=99.66%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [107M/0K /s] [104/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9659
  read : io=6,492MB, bw=108MB/s, iops=108, runt= 60008msec
    clat (msec): min=7, max=58, avg= 9.24, stdev= 2.38
    bw (KB/s) : min=96864, max=116502, per=100.10%, avg=110887.51, stdev=4865.04
  cpu          : usr=0.04%, sys=1.72%, ctx=6541, majf=0, minf=281
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=6492/0, short=0/0

     lat (msec): 10=95.84%, 20=3.43%, 50=0.68%, 100=0.05%

Run status group 0 (all jobs):
   READ: io=6,492MB, aggrb=108MB/s, minb=111MB/s, maxb=111MB/s, mint=60008msec, maxt=60008msec

Disk stats (read/write):
  sdd: ios=12962/0, merge=0/0, ticks=88054/0, in_queue=88066, util=99.41%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [62M/0K /s] [60/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9664
  read : io=3,631MB, bw=61,960KB/s, iops=60, runt= 60009msec
    clat (msec): min=8, max=81, avg=16.52, stdev= 4.06
    bw (KB/s) : min=50927, max=67603, per=100.09%, avg=62016.05, stdev=2780.71
  cpu          : usr=0.03%, sys=0.98%, ctx=3659, majf=0, minf=289
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3631/0, short=0/0

     lat (msec): 10=0.17%, 20=90.14%, 50=9.58%, 100=0.11%

Run status group 0 (all jobs):
   READ: io=3,631MB, aggrb=61,959KB/s, minb=63,446KB/s, maxb=63,446KB/s, mint=60009msec, maxt=60009msec

Disk stats (read/write):
  sdd: ios=7250/0, merge=0/0, ticks=101864/0, in_queue=101879, util=99.69%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [40M/0K /s] [38/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9669
  read : io=2,463MB, bw=42,027KB/s, iops=41, runt= 60012msec
    clat (msec): min=17, max=76, avg=24.36, stdev= 4.56
    bw (KB/s) : min=37200, max=46180, per=100.11%, avg=42074.16, stdev=1709.54
  cpu          : usr=0.03%, sys=0.63%, ctx=2476, majf=0, minf=279
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2463/0, short=0/0

     lat (msec): 20=10.23%, 50=89.44%, 100=0.32%

Run status group 0 (all jobs):
   READ: io=2,463MB, aggrb=42,026KB/s, minb=43,035KB/s, maxb=43,035KB/s, mint=60012msec, maxt=60012msec

Disk stats (read/write):
  sdd: ios=4917/0, merge=0/0, ticks=98300/0, in_queue=98317, util=99.83%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [43M/0K /s] [41/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9703
  read : io=2,522MB, bw=43,040KB/s, iops=42, runt= 60003msec
    clat (msec): min=12, max=73, avg=23.79, stdev= 5.05
    bw (KB/s) : min=37559, max=48858, per=100.11%, avg=43088.15, stdev=2171.70
  cpu          : usr=0.02%, sys=0.68%, ctx=2546, majf=0, minf=342
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2522/0, short=0/0

     lat (msec): 20=21.81%, 50=77.99%, 100=0.20%

Run status group 0 (all jobs):
   READ: io=2,522MB, aggrb=43,039KB/s, minb=44,072KB/s, maxb=44,072KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdd: ios=5036/0, merge=0/0, ticks=104875/0, in_queue=104898, util=99.82%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing read tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [3,724K/0K /s] [7K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9711
  read : io=215MB, bw=3,668KB/s, iops=7,335, runt= 60001msec
    clat (usec): min=186, max=21,584, avg=543.12, stdev=64.30
    bw (KB/s) : min=  885, max=  964, per=25.02%, avg=917.41, stdev= 6.01
  cpu          : usr=0.47%, sys=0.83%, ctx=440331, majf=0, minf=101
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=440158/0, short=0/0
     lat (usec): 250=0.01%, 500=18.25%, 750=81.51%, 1000=0.19%
     lat (msec): 2=0.03%, 4=0.01%, 10=0.02%, 20=0.01%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=215MB, aggrb=3,667KB/s, minb=3,755KB/s, maxb=3,755KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=438387/0, merge=1086/0, ticks=210191/0, in_queue=210264, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [3,698K/0K /s] [7K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9719
  read : io=215MB, bw=3,677KB/s, iops=7,353, runt= 60001msec
    clat (usec): min=170, max=16,669, avg=541.80, stdev=57.98
    bw (KB/s) : min=  886, max= 1138, per=25.02%, avg=919.63, stdev=11.59
  cpu          : usr=0.43%, sys=0.77%, ctx=441363, majf=0, minf=116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=441208/0, short=0/0
     lat (usec): 250=0.01%, 500=14.61%, 750=85.19%, 1000=0.16%
     lat (msec): 2=0.02%, 4=0.01%, 10=0.02%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=215MB, aggrb=3,676KB/s, minb=3,764KB/s, maxb=3,764KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=438866/0, merge=1628/0, ticks=209914/0, in_queue=209966, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [0.0% done] [41K/0K /s] [81/0 iops] [eta 557d:22h:39m:41s]
file1: (groupid=0, jobs=4): err= 0: pid=9727
  read : io=2,501KB, bw=42,505B/s, iops=83, runt= 60239msec
    clat (msec): min=2, max=434, avg=48.13, stdev=50.71
    bw (KB/s) : min=    5, max=   20, per=25.87%, avg=10.61, stdev= 1.99
  cpu          : usr=0.04%, sys=0.12%, ctx=5010, majf=0, minf=156
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5001/0, short=0/0

     lat (msec): 4=0.62%, 10=25.43%, 20=61.55%, 50=1.14%, 100=0.02%
     lat (msec): 250=0.10%, 500=11.14%

Run status group 0 (all jobs):
   READ: io=2,500KB, aggrb=41KB/s, minb=42KB/s, maxb=42KB/s, mint=60239msec, maxt=60239msec

Disk stats (read/write):
  sdd: ios=4986/0, merge=0/0, ticks=238979/0, in_queue=239581, util=99.96%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 3 (f=3): [rrr_] [50.4% done] [64K/0K /s] [125/0 iops] [eta 01m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9735
  read : io=3,930KB, bw=66,914B/s, iops=130, runt= 60141msec
    clat (msec): min=1, max=429, avg=30.58, stdev=40.91
    bw (KB/s) : min=    8, max=   30, per=26.19%, avg=17.02, stdev= 3.52
  cpu          : usr=0.06%, sys=0.23%, ctx=7872, majf=0, minf=2575
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7860/0, short=0/0

     lat (msec): 2=0.17%, 4=6.90%, 10=64.90%, 20=20.69%, 50=0.04%
     lat (msec): 250=0.14%, 500=7.18%

Run status group 0 (all jobs):
   READ: io=3,930KB, aggrb=65KB/s, minb=66KB/s, maxb=66KB/s, mint=60141msec, maxt=60141msec

Disk stats (read/write):
  sdd: ios=7835/0, merge=0/0, ticks=239158/0, in_queue=239437, util=99.93%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [130M/0K /s] [126/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9748
  read : io=7,275MB, bw=121MB/s, iops=121, runt= 60033msec
    clat (msec): min=4, max=254, avg=119.38, stdev= 5.38
    bw (KB/s) : min= 5657, max=44000, per=12.00%, avg=14892.86, stdev=656.38
  cpu          : usr=0.02%, sys=0.29%, ctx=7310, majf=0, minf=1124
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=7275/0, short=0/0

     lat (msec): 10=0.18%, 20=15.00%, 50=78.53%, 100=0.91%, 250=5.37%
     lat (msec): 500=0.01%

Run status group 0 (all jobs):
   READ: io=7,275MB, aggrb=121MB/s, minb=124MB/s, maxb=124MB/s, mint=60033msec, maxt=60033msec

Disk stats (read/write):
  sdd: ios=14520/0, merge=0/0, ticks=453447/0, in_queue=453728, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [122M/0K /s] [119/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9758
  read : io=6,743MB, bw=112MB/s, iops=112, runt= 60029msec
    clat (msec): min=8, max=240, avg=35.58, stdev=18.26
    bw (KB/s) : min= 9925, max=45511, per=25.20%, avg=28988.59, stdev=3350.21
  cpu          : usr=0.02%, sys=0.27%, ctx=6771, majf=0, minf=1132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=6743/0, short=0/0

     lat (msec): 10=28.07%, 20=16.91%, 50=36.78%, 100=4.30%, 250=13.94%

Run status group 0 (all jobs):
   READ: io=6,743MB, aggrb=112MB/s, minb=115MB/s, maxb=115MB/s, mint=60029msec, maxt=60029msec

Disk stats (read/write):
  sdd: ios=13459/0, merge=0/0, ticks=447720/0, in_queue=448037, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=read --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [166M/0K /s] [162/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9766
  read : io=9,330MB, bw=155MB/s, iops=155, runt= 60015msec
    clat (msec): min=6, max=264, avg=25.73, stdev=17.11
    bw (KB/s) : min=25830, max=56440, per=25.16%, avg=40050.92, stdev=3377.52
  cpu          : usr=0.03%, sys=0.39%, ctx=9385, majf=0, minf=1124
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=9330/0, short=0/0

     lat (msec): 10=43.33%, 20=34.19%, 50=10.12%, 100=2.19%, 250=10.16%
     lat (msec): 500=0.01%

Run status group 0 (all jobs):
   READ: io=9,330MB, aggrb=155MB/s, minb=159MB/s, maxb=159MB/s, mint=60015msec, maxt=60015msec

Disk stats (read/write):
  sdd: ios=18627/0, merge=0/0, ticks=447458/0, in_queue=447768, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [61M/0K /s] [59/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9774
  read : io=3,622MB, bw=61,758KB/s, iops=60, runt= 60056msec
    clat (msec): min=11, max=344, avg=60.09, stdev=10.74
    bw (KB/s) : min= 3079, max=31813, per=29.57%, avg=18263.97, stdev=1224.68
  cpu          : usr=0.01%, sys=0.13%, ctx=3637, majf=0, minf=1156
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=3622/0, short=0/0

     lat (msec): 20=2.60%, 50=38.60%, 100=44.73%, 250=13.83%, 500=0.25%

Run status group 0 (all jobs):
   READ: io=3,622MB, aggrb=61,757KB/s, minb=63,240KB/s, maxb=63,240KB/s, mint=60056msec, maxt=60056msec

Disk stats (read/write):
  sdd: ios=7226/0, merge=0/0, ticks=457661/0, in_queue=457960, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [44M/0K /s] [42/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9811
  read : io=2,485MB, bw=42,350KB/s, iops=41, runt= 60086msec
    clat (msec): min=16, max=224, avg=99.25, stdev=20.45
    bw (KB/s) : min= 5485, max=25748, per=24.68%, avg=10449.69, stdev=2075.04
  cpu          : usr=0.00%, sys=0.10%, ctx=2500, majf=0, minf=1116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2485/0, short=0/0

     lat (msec): 20=0.44%, 50=3.38%, 100=62.13%, 250=34.04%

Run status group 0 (all jobs):
   READ: io=2,485MB, aggrb=42,349KB/s, minb=43,366KB/s, maxb=43,366KB/s, mint=60086msec, maxt=60086msec

Disk stats (read/write):
  sdd: ios=4954/0, merge=0/0, ticks=455123/0, in_queue=455596, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randread --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [43M/0K /s] [41/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9819
  read : io=2,521MB, bw=42,976KB/s, iops=41, runt= 60068msec
    clat (msec): min=14, max=368, avg=78.07, stdev=11.33
    bw (KB/s) : min= 3555, max=21222, per=31.64%, avg=13596.13, stdev=774.14
  cpu          : usr=0.01%, sys=0.09%, ctx=2534, majf=0, minf=1474
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=2521/0, short=0/0

     lat (msec): 20=0.40%, 50=1.35%, 100=64.30%, 250=33.48%, 500=0.48%

Run status group 0 (all jobs):
   READ: io=2,521MB, aggrb=42,976KB/s, minb=44,007KB/s, maxb=44,007KB/s, mint=60068msec, maxt=60068msec

Disk stats (read/write):
  sdd: ios=5028/0, merge=0/0, ticks=461011/0, in_queue=461451, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/4,743K /s] [0/9K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9830
  write: io=270MB, bw=4,610KB/s, iops=9,219, runt= 60000msec
    clat (usec): min=79, max=9,173, avg=106.32, stdev=26.85
    bw (KB/s) : min= 2844, max= 4711, per=100.02%, avg=4609.82, stdev=182.80
  cpu          : usr=2.65%, sys=50.99%, ctx=553344, majf=0, minf=24
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/553148, short=0/0
     lat (usec): 100=37.76%, 250=62.12%, 500=0.03%, 750=0.04%, 1000=0.04%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
  WRITE: io=270MB, aggrb=4,609KB/s, minb=4,720KB/s, maxb=4,720KB/s, mint=60000msec, maxt=60000msec

Disk stats (read/write):
  sdd: ios=0/552357, merge=0/0, ticks=0/54627, in_queue=54578, util=91.27%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/4,770K /s] [0/9K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9835
  write: io=271MB, bw=4,626KB/s, iops=9,251, runt= 60001msec
    clat (usec): min=81, max=12,168, avg=105.96, stdev=27.17
    bw (KB/s) : min= 2860, max= 4703, per=100.02%, avg=4625.76, stdev=197.02
  cpu          : usr=2.52%, sys=51.38%, ctx=555252, majf=0, minf=29
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/555070, short=0/0
     lat (usec): 100=39.13%, 250=60.76%, 500=0.03%, 750=0.04%, 1000=0.04%
     lat (msec): 2=0.01%, 4=0.01%, 20=0.01%

Run status group 0 (all jobs):
  WRITE: io=271MB, aggrb=4,625KB/s, minb=4,736KB/s, maxb=4,736KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=0/554254, merge=0/0, ticks=0/54862, in_queue=54820, util=91.60%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/89K /s] [0/174 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9840
  write: io=5,380KB, bw=91,814B/s, iops=179, runt= 60003msec
    clat (usec): min=156, max=17,518, avg=5568.46, stdev=2189.18
    bw (KB/s) : min=   75, max=  220, per=100.35%, avg=89.31, stdev=13.16
  cpu          : usr=0.16%, sys=1.33%, ctx=10770, majf=0, minf=223
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/10760, short=0/0
     lat (usec): 250=1.31%, 500=0.02%, 750=0.01%, 1000=0.08%
     lat (msec): 2=0.03%, 4=11.57%, 10=81.58%, 20=5.40%

Run status group 0 (all jobs):
  WRITE: io=5,380KB, aggrb=89KB/s, minb=91KB/s, maxb=91KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdd: ios=0/10727, merge=0/0, ticks=0/59576, in_queue=59574, util=99.56%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=50010786201 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/127K /s] [0/249 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9845
  write: io=7,444KB, bw=124KB/s, iops=248, runt= 60001msec
    clat (usec): min=158, max=15,443, avg=4021.50, stdev=1896.76
    bw (KB/s) : min=  107, max=  258, per=99.82%, avg=123.77, stdev=13.51
  cpu          : usr=0.24%, sys=1.67%, ctx=14910, majf=0, minf=2485
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/14888, short=0/0
     lat (usec): 250=0.95%, 500=0.03%, 750=0.26%, 1000=0.56%
     lat (msec): 2=0.04%, 4=66.05%, 10=27.45%, 20=4.66%

Run status group 0 (all jobs):
  WRITE: io=7,444KB, aggrb=124KB/s, minb=127KB/s, maxb=127KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=0/14843, merge=0/0, ticks=0/59632, in_queue=59630, util=99.63%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/112M /s] [0/109 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9850
  write: io=5,120MB, bw=111MB/s, iops=111, runt= 45977msec
    clat (msec): min=4, max=25, avg= 8.98, stdev= 1.05
    bw (KB/s) : min=109280, max=118784, per=100.09%, avg=114137.80, stdev=2130.92
  cpu          : usr=0.07%, sys=2.84%, ctx=5202, majf=0, minf=23
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0

     lat (msec): 10=97.79%, 20=2.19%, 50=0.02%

Run status group 0 (all jobs):
  WRITE: io=5,120MB, aggrb=111MB/s, minb=114MB/s, maxb=114MB/s, mint=45977msec, maxt=45977msec

Disk stats (read/write):
  sdd: ios=0/10211, merge=0/0, ticks=0/67117, in_queue=67117, util=99.46%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/61M /s] [0/59 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9855
  write: io=3,553MB, bw=60,632KB/s, iops=59, runt= 60006msec
    clat (msec): min=4, max=41, avg=16.88, stdev= 3.17
    bw (KB/s) : min=52409, max=64758, per=100.12%, avg=60701.21, stdev=2648.79
  cpu          : usr=0.02%, sys=1.58%, ctx=3620, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/3553, short=0/0

     lat (msec): 10=0.03%, 20=90.06%, 50=9.91%

Run status group 0 (all jobs):
  WRITE: io=3,553MB, aggrb=60,631KB/s, minb=62,086KB/s, maxb=62,086KB/s, mint=60006msec, maxt=60006msec

Disk stats (read/write):
  sdd: ios=0/7094, merge=0/0, ticks=0/88615, in_queue=88614, util=99.83%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/114M /s] [0/111 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9862
  write: io=6,722MB, bw=112MB/s, iops=112, runt= 60007msec
    clat (usec): min=4,563, max=19,843, avg=8923.34, stdev=802.09
    bw (KB/s) : min=109714, max=118784, per=100.10%, avg=114819.53, stdev=2024.44
  cpu          : usr=0.06%, sys=2.81%, ctx=6809, majf=0, minf=24
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/6722, short=0/0

     lat (msec): 10=98.42%, 20=1.58%

Run status group 0 (all jobs):
  WRITE: io=6,722MB, aggrb=112MB/s, minb=115MB/s, maxb=115MB/s, mint=60007msec, maxt=60007msec

Disk stats (read/write):
  sdd: ios=0/13424, merge=0/0, ticks=0/87738, in_queue=87732, util=99.53%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/82M /s] [0/79 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9896
  write: io=4,591MB, bw=78,349KB/s, iops=76, runt= 60003msec
    clat (msec): min=4, max=44, avg=13.06, stdev= 3.21
    bw (KB/s) : min=71536, max=94368, per=100.06%, avg=78397.58, stdev=2890.44
  cpu          : usr=0.05%, sys=1.98%, ctx=4655, majf=0, minf=28
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/4591, short=0/0

     lat (msec): 10=5.40%, 20=90.63%, 50=3.96%

Run status group 0 (all jobs):
  WRITE: io=4,591MB, aggrb=78,349KB/s, minb=80,229KB/s, maxb=80,229KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdd: ios=0/9163, merge=0/0, ticks=0/70250, in_queue=70275, util=99.68%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/50M /s] [0/48 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9901
  write: io=2,874MB, bw=49,039KB/s, iops=47, runt= 60013msec
    clat (msec): min=4, max=49, avg=20.88, stdev= 3.81
    bw (KB/s) : min=43743, max=73434, per=100.14%, avg=49107.43, stdev=2919.76
  cpu          : usr=0.03%, sys=1.23%, ctx=2917, majf=0, minf=25
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2874, short=0/0

     lat (msec): 10=1.32%, 20=42.41%, 50=56.26%

Run status group 0 (all jobs):
  WRITE: io=2,874MB, aggrb=49,038KB/s, minb=50,215KB/s, maxb=50,215KB/s, mint=60013msec, maxt=60013msec

Disk stats (read/write):
  sdd: ios=0/5739, merge=0/0, ticks=0/66006, in_queue=66011, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=500107862016 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/53M /s] [0/51 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=9906
  write: io=3,091MB, bw=52,749KB/s, iops=51, runt= 60005msec
    clat (msec): min=4, max=45, avg=19.41, stdev= 3.92
    bw (KB/s) : min=46637, max=76000, per=100.12%, avg=52809.12, stdev=3489.84
  cpu          : usr=0.04%, sys=1.36%, ctx=3137, majf=0, minf=102
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/3091, short=0/0

     lat (msec): 10=0.49%, 20=66.97%, 50=32.55%

Run status group 0 (all jobs):
  WRITE: io=3,091MB, aggrb=52,748KB/s, minb=54,014KB/s, maxb=54,014KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdd: ios=0/6173, merge=0/0, ticks=0/66239, in_queue=66242, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/2,964K /s] [0/6K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9912
  write: io=187MB, bw=3,194KB/s, iops=6,387, runt= 60001msec
    clat (usec): min=240, max=34,339, avg=628.35, stdev=155.21
    bw (KB/s) : min=  611, max= 1468, per=24.87%, avg=794.06, stdev=92.93
  cpu          : usr=0.35%, sys=0.82%, ctx=383428, majf=0, minf=97
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/383260, short=0/0
     lat (usec): 250=0.01%, 500=38.53%, 750=32.82%, 1000=18.36%
     lat (msec): 2=10.29%, 4=0.01%, 20=0.01%, 50=0.01%

Run status group 0 (all jobs):
  WRITE: io=187MB, aggrb=3,193KB/s, minb=3,270KB/s, maxb=3,270KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=0/345600, merge=0/37130, ticks=0/208450, in_queue=208473, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/2,979K /s] [0/6K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9922
  write: io=228MB, bw=3,893KB/s, iops=7,785, runt= 60001msec
    clat (usec): min=226, max=32,127, avg=513.26, stdev=156.71
    bw (KB/s) : min=  637, max= 1464, per=24.97%, avg=972.00, stdev=119.45
  cpu          : usr=0.52%, sys=1.10%, ctx=467335, majf=0, minf=107
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/467112, short=0/0
     lat (usec): 250=0.01%, 500=60.02%, 750=26.56%, 1000=9.40%
     lat (msec): 2=4.00%, 10=0.01%, 20=0.01%, 50=0.01%

Run status group 0 (all jobs):
  WRITE: io=228MB, aggrb=3,892KB/s, minb=3,985KB/s, maxb=3,985KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdd: ios=0/342856, merge=0/123696, ticks=0/173031, in_queue=173102, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 3 (f=3): [www_] [50.4% done] [0K/92K /s] [0/180 iops] [eta 01m:00s] 
file1: (groupid=0, jobs=4): err= 0: pid=9933
  write: io=5,372KB, bw=91,644B/s, iops=178, runt= 60019msec
    clat (usec): min=149, max=49,727, avg=21723.89, stdev=3239.05
    bw (KB/s) : min=   18, max=  174, per=25.28%, avg=22.50, stdev= 6.12
  cpu          : usr=0.07%, sys=0.06%, ctx=10759, majf=0, minf=387
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/10743, short=0/0
     lat (usec): 250=1.28%, 500=0.05%, 1000=0.01%
     lat (msec): 2=0.05%, 4=0.01%, 10=0.29%, 20=32.19%, 50=66.13%

Run status group 0 (all jobs):
  WRITE: io=5,371KB, aggrb=89KB/s, minb=91KB/s, maxb=91KB/s, mint=60019msec, maxt=60019msec

Disk stats (read/write):
  sdd: ios=0/10713, merge=0/0, ticks=0/238820, in_queue=238854, util=99.93%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=512 --size=50010786201 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 3 (f=3): [_www] [50.4% done] [0K/125K /s] [0/244 iops] [eta 01m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9941
  write: io=7,422KB, bw=124KB/s, iops=247, runt= 60018msec
    clat (usec): min=147, max=39,843, avg=16313.88, stdev=1907.00
    bw (KB/s) : min=   27, max=  192, per=24.47%, avg=30.10, stdev= 1.67
  cpu          : usr=0.08%, sys=0.09%, ctx=14864, majf=0, minf=3298
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/14843, short=0/0
     lat (usec): 250=0.98%, 750=0.01%
     lat (msec): 2=0.01%, 4=0.14%, 10=0.42%, 20=81.06%, 50=17.38%

Run status group 0 (all jobs):
  WRITE: io=7,421KB, aggrb=123KB/s, minb=126KB/s, maxb=126KB/s, mint=60018msec, maxt=60018msec

Disk stats (read/write):
  sdd: ios=0/14802, merge=0/0, ticks=0/238595, in_queue=238616, util=99.95%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/95M /s] [0/92 iops] [eta 00m:00s] 
file1: (groupid=0, jobs=4): err= 0: pid=9949
  write: io=5,816MB, bw=99,192KB/s, iops=96, runt= 60041msec
    clat (msec): min=17, max=77, avg=41.28, stdev= 3.64
    bw (KB/s) : min=22393, max=27945, per=25.03%, avg=24824.70, stdev=356.75
  cpu          : usr=0.01%, sys=0.17%, ctx=5842, majf=0, minf=96
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5816, short=0/0

     lat (msec): 20=6.26%, 50=79.04%, 100=14.70%

Run status group 0 (all jobs):
  WRITE: io=5,816MB, aggrb=99,191KB/s, minb=99MB/s, maxb=99MB/s, mint=60041msec, maxt=60041msec

Disk stats (read/write):
  sdd: ios=0/11604, merge=0/0, ticks=0/454643, in_queue=454886, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/69M /s] [0/67 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=9957
  write: io=4,170MB, bw=71,117KB/s, iops=69, runt= 60043msec
    clat (msec): min=4, max=136, avg=57.59, stdev=11.71
    bw (KB/s) : min=12214, max=21703, per=25.05%, avg=17812.85, stdev=872.23
  cpu          : usr=0.01%, sys=0.13%, ctx=4185, majf=0, minf=104
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/4170, short=0/0

     lat (msec): 10=0.12%, 20=9.30%, 50=28.59%, 100=57.77%, 250=4.22%

Run status group 0 (all jobs):
  WRITE: io=4,170MB, aggrb=71,117KB/s, minb=72,823KB/s, maxb=72,823KB/s, mint=60043msec, maxt=60043msec

Disk stats (read/write):
  sdd: ios=0/8319, merge=0/0, ticks=0/460354, in_queue=460451, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=write --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/98M /s] [0/95 iops] [eta 00m:00s] 
file1: (groupid=0, jobs=4): err= 0: pid=9996
  write: io=5,757MB, bw=98,199KB/s, iops=95, runt= 60033msec
    clat (msec): min=5, max=74, avg=41.70, stdev= 3.67
    bw (KB/s) : min=22671, max=26570, per=25.02%, avg=24568.86, stdev=268.64
  cpu          : usr=0.01%, sys=0.18%, ctx=5776, majf=0, minf=96
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5757, short=0/0

     lat (msec): 10=0.03%, 20=5.45%, 50=78.69%, 100=15.82%

Run status group 0 (all jobs):
  WRITE: io=5,757MB, aggrb=98,198KB/s, minb=98MB/s, maxb=98MB/s, mint=60033msec, maxt=60033msec

Disk stats (read/write):
  sdd: ios=0/11490, merge=0/0, ticks=0/454559, in_queue=454703, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/81M /s] [0/78 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=10004
  write: io=4,591MB, bw=78,281KB/s, iops=76, runt= 60055msec
    clat (msec): min=7, max=79, avg=52.28, stdev= 2.49
    bw (KB/s) : min=17355, max=26624, per=25.05%, avg=19612.75, stdev=440.26
  cpu          : usr=0.01%, sys=0.12%, ctx=4603, majf=0, minf=112
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/4591, short=0/0

     lat (msec): 10=0.02%, 20=0.22%, 50=19.86%, 100=79.90%

Run status group 0 (all jobs):
  WRITE: io=4,591MB, aggrb=78,281KB/s, minb=80,160KB/s, maxb=80,160KB/s, mint=60055msec, maxt=60055msec

Disk stats (read/write):
  sdd: ios=0/9162, merge=0/0, ticks=0/428464, in_queue=428646, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=494739152896 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/48M /s] [0/46 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=10012
  write: io=2,901MB, bw=49,457KB/s, iops=48, runt= 60065msec
    clat (msec): min=18, max=129, avg=82.78, stdev= 3.97
    bw (KB/s) : min=11314, max=18652, per=25.01%, avg=12370.55, stdev=334.84
  cpu          : usr=0.01%, sys=0.09%, ctx=2916, majf=0, minf=96
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/2901, short=0/0

     lat (msec): 20=0.34%, 50=0.17%, 100=97.62%, 250=1.86%

Run status group 0 (all jobs):
  WRITE: io=2,901MB, aggrb=49,456KB/s, minb=50,643KB/s, maxb=50,643KB/s, mint=60065msec, maxt=60065msec

Disk stats (read/write):
  sdd: ios=0/5786, merge=0/0, ticks=0/424496, in_queue=424755, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdd --direct=1 --rw=randwrite --bs=1m --size=500107862016 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/55M /s] [0/53 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=10020
  write: io=3,101MB, bw=52,859KB/s, iops=51, runt= 60074msec
    clat (msec): min=20, max=121, avg=77.48, stdev= 5.43
    bw (KB/s) : min=11592, max=20197, per=25.03%, avg=13230.44, stdev=425.92
  cpu          : usr=0.00%, sys=0.08%, ctx=3114, majf=0, minf=337
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/3101, short=0/0

     lat (msec): 50=0.55%, 100=96.29%, 250=3.16%

Run status group 0 (all jobs):
  WRITE: io=3,101MB, aggrb=52,858KB/s, minb=54,127KB/s, maxb=54,127KB/s, mint=60074msec, maxt=60074msec

Disk stats (read/write):
  sdd: ios=0/6185, merge=0/0, ticks=0/424704, in_queue=424918, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
[root@iotest ~]#