wfischer@wfischer-t410-ubuntu:~$ cat fiotester2.sh 
#!/bin/bash

FIO="/usr/local/bin/fio"
FN="/dev/sdb"
OFFSET="154673176576"
# 5GB are 5.368.709.120 Bytes
DEVSIZE="160041885696"
DEVSIZE10P="16004188569"
BSIOPS="4k"
BSTHR="1m"

$FIO --version

hdparm -W0 /dev/sdb
hdparm -I /dev/sdb

for TESTCASE in "read" "write"
do 
	for NJ in 1 4
	do
		echo "#################################################################################"
		echo "Executing $TESTCASE tests with numjobs=$NJ"
		for testrun in `seq 1 10`
		do
			case $testrun in
				1)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				2)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				3)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				4)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				5)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				6)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				7)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				8)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
				9)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";;
				10)	TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";;
			esac
			echo "---------------------------------------------------------------------------------"
			echo "Executing test $testrun: $TESTCMD"
			$TESTCMD
			echo "Sleeping for 30 seconds... ------------------------------------------------------"
			sleep 30
		done
	done
done
wfischer@wfischer-t410-ubuntu:~$ sudo /home/wfischer/fiotester2.sh 
[sudo] password for wfischer: 
fio 1.41

/dev/sdb:
 setting drive write-caching to 0 (off)
 write-caching =  0 (off)

/dev/sdb:

ATA device, with non-removable media
	Model Number:       INTEL SSDSA2M160G2GC                    
	Serial Number:      CVPO014602FC160AGN  
	Firmware Revision:  2CV102HD
	Transport:          Serial, ATA8-AST, SATA 1.0a, SATA II Extensions, SATA Rev 2.5, SATA Rev 2.6
Standards:
	Used: ATA/ATAPI-7 T13 1532D revision 1 
	Supported: 7 6 5 4 
Configuration:
	Logical		max	current
	cylinders	16383	16383
	heads		16	16
	sectors/track	63	63
	--
	CHS current addressable sectors:   16514064
	LBA    user addressable sectors:  268435455
	LBA48  user addressable sectors:  312581808
	Logical  Sector size:                   512 bytes
	Physical Sector size:                   512 bytes
	device size with M = 1024*1024:      152627 MBytes
	device size with M = 1000*1000:      160041 MBytes (160 GB)
	cache/buffer size  = unknown
	Nominal Media Rotation Rate: Solid State Device
Capabilities:
	LBA, IORDY(can be disabled)
	Queue depth: 32
	Standby timer values: spec'd by Standard, no device specific minimum
	R/W multiple sector transfer: Max = 16	Current = 1
	DMA: mdma0 mdma1 mdma2 udma0 udma1 udma2 udma3 udma4 udma5 *udma6 
	     Cycle time: min=120ns recommended=120ns
	PIO: pio0 pio1 pio2 pio3 pio4 
	     Cycle time: no flow control=120ns  IORDY flow control=120ns
Commands/features:
	Enabled	Supported:
	   *	SMART feature set
	    	Security Mode feature set
	   *	Power Management feature set
	    	Write cache
	   *	Look-ahead
	   *	Host Protected Area feature set
	   *	WRITE_BUFFER command
	   *	READ_BUFFER command
	   *	NOP cmd
	   *	DOWNLOAD_MICROCODE
	    	SET_MAX security extension
	   *	48-bit Address feature set
	   *	Device Configuration Overlay feature set
	   *	Mandatory FLUSH_CACHE
	   *	FLUSH_CACHE_EXT
	   *	SMART error logging
	   *	SMART self-test
	   *	General Purpose Logging feature set
	   *	WRITE_{DMA|MULTIPLE}_FUA_EXT
	   *	64-bit World wide name
	   *	IDLE_IMMEDIATE with UNLOAD
	   *	WRITE_UNCORRECTABLE_EXT command
	   *	{READ,WRITE}_DMA_EXT_GPL commands
	   *	Segmented DOWNLOAD_MICROCODE
	   *	Gen1 signaling speed (1.5Gb/s)
	   *	Gen2 signaling speed (3.0Gb/s)
	   *	Native Command Queueing (NCQ)
	   *	Phy event counters
	    	Device-initiated interface power management
	   *	Software settings preservation
	   *	Data Set Management determinate TRIM supported
Security: 
	Master password revision code = 65534
		supported
	not	enabled
	not	locked
	not	frozen
	not	expired: security count
		supported: enhanced erase
	2min for SECURITY ERASE UNIT. 2min for ENHANCED SECURITY ERASE UNIT.
Logical Unit WWN Device Identifier: 500151795928e489
	NAA		: 5
	IEEE OUI	: 001517
	Unique ID	: 95928e489
Checksum: correct
#################################################################################
Executing read tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=4k --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [39387K/0K /s] [9616/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2475
  read : io=2251MB, bw=38410KB/s, iops=9602, runt= 60001msec
    clat (usec): min=52, max=2995, avg=99.92, stdev=19.13
    bw (KB/s) : min=36904, max=44848, per=100.01%, avg=38413.92, stdev=1176.26
  cpu          : usr=5.44%, sys=22.74%, ctx=577980, majf=0, minf=31
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=576156/0, short=0/0
     lat (usec): 100=33.00%, 250=66.73%, 500=0.27%, 750=0.01%, 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%

Run status group 0 (all jobs):
   READ: io=2251MB, aggrb=38409KB/s, minb=39331KB/s, maxb=39331KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=575353/0, merge=0/0, ticks=50608/0, in_queue=50440, util=84.07%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=4k --size=16004188569 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [39325K/0K /s] [9601/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2478
  read : io=3057MB, bw=52164KB/s, iops=13041, runt= 60001msec
    clat (usec): min=39, max=2836, avg=73.99, stdev=23.21
    bw (KB/s) : min=36864, max=72496, per=100.21%, avg=52272.13, stdev=10463.03
  cpu          : usr=4.45%, sys=17.81%, ctx=784126, majf=0, minf=32
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=782477/0, short=0/0
     lat (usec): 50=1.75%, 100=78.17%, 250=19.86%, 500=0.22%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%

Run status group 0 (all jobs):
   READ: io=3057MB, aggrb=52164KB/s, minb=53416KB/s, maxb=53416KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=781806/0, merge=0/0, ticks=48024/0, in_queue=47560, util=79.27%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [38273K/0K /s] [9344/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2624
  read : io=2477MB, bw=42269KB/s, iops=10567, runt= 60001msec
    clat (usec): min=45, max=3803, avg=89.38, stdev=25.24
    bw (KB/s) : min=36096, max=47536, per=100.12%, avg=42316.91, stdev=4593.96
  cpu          : usr=6.33%, sys=22.39%, ctx=636579, majf=0, minf=2414
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=634041/0, short=0/0
     lat (usec): 50=0.01%, 100=49.00%, 250=50.74%, 500=0.24%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%

Run status group 0 (all jobs):
   READ: io=2477MB, aggrb=42268KB/s, minb=43283KB/s, maxb=43283KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=633250/0, merge=0/0, ticks=48504/0, in_queue=48200, util=80.34%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --size=16004188569 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [40357K/0K /s] [9853/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2627
  read : io=2228MB, bw=38020KB/s, iops=9505, runt= 60001msec
    clat (usec): min=40, max=4362, avg=99.45, stdev=24.60
    bw (KB/s) : min=35992, max=49128, per=100.04%, avg=38037.04, stdev=2569.07
  cpu          : usr=7.46%, sys=26.01%, ctx=572447, majf=0, minf=268
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=570316/0, short=0/0
     lat (usec): 50=0.05%, 100=28.27%, 250=71.40%, 500=0.26%, 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=2228MB, aggrb=38020KB/s, minb=38932KB/s, maxb=38932KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=569567/0, merge=0/0, ticks=48368/0, in_queue=48140, util=80.24%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [95.2% done] [258M/0K /s] [252/0 iops] [eta 00m:01s]
file1: (groupid=0, jobs=1): err= 0: pid=2630
  read : io=1024MB, bw=260477KB/s, iops=254, runt= 20128msec
    clat (usec): min=3828, max=5057, avg=3925.74, stdev=55.15
    bw (KB/s) : min=257019, max=262144, per=100.12%, avg=260780.90, stdev=1793.41
  cpu          : usr=0.28%, sys=5.62%, ctx=5296, majf=0, minf=284
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0

     lat (msec): 4=98.40%, 10=1.60%

Run status group 0 (all jobs):
   READ: io=5120MB, aggrb=260476KB/s, minb=266728KB/s, maxb=266728KB/s, mint=20128msec, maxt=20128msec

Disk stats (read/write):
  sdb: ios=10126/0, merge=0/0, ticks=29364/0, in_queue=29344, util=96.27%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [261M/0K /s] [255/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2633
  read : io=1024MB, bw=259407KB/s, iops=253, runt= 20211msec
    clat (usec): min=3828, max=5938, avg=3941.98, stdev=65.81
    bw (KB/s) : min=256508, max=262144, per=100.10%, avg=259654.35, stdev=2137.28
  cpu          : usr=0.20%, sys=4.91%, ctx=5274, majf=0, minf=284
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0

     lat (msec): 4=97.99%, 10=2.01%

Run status group 0 (all jobs):
   READ: io=5120MB, aggrb=259407KB/s, minb=265633KB/s, maxb=265633KB/s, mint=20211msec, maxt=20211msec

Disk stats (read/write):
  sdb: ios=10211/0, merge=0/0, ticks=29852/0, in_queue=29812, util=97.05%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [262M/0K /s] [256/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2636
  read : io=2920MB, bw=259537KB/s, iops=253, runt= 60003msec
    clat (usec): min=3821, max=7818, avg=3940.03, stdev=72.71
    bw (KB/s) : min=257019, max=262616, per=100.09%, avg=259762.15, stdev=2067.55
  cpu          : usr=0.17%, sys=4.83%, ctx=15646, majf=0, minf=287
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15208/0, short=0/0

     lat (msec): 4=97.71%, 10=2.29%

Run status group 0 (all jobs):
   READ: io=15208MB, aggrb=259536KB/s, minb=265765KB/s, maxb=265765KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdb: ios=30377/0, merge=0/0, ticks=86984/0, in_queue=86844, util=96.37%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [262M/0K /s] [256/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2643
  read : io=1024MB, bw=258334KB/s, iops=252, runt= 20295msec
    clat (usec): min=3827, max=5042, avg=3955.62, stdev=55.21
    bw (KB/s) : min=257019, max=262144, per=100.10%, avg=258580.45, stdev=1341.27
  cpu          : usr=0.34%, sys=4.43%, ctx=5251, majf=0, minf=283
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0

     lat (msec): 4=97.99%, 10=2.01%

Run status group 0 (all jobs):
   READ: io=5120MB, aggrb=258333KB/s, minb=264533KB/s, maxb=264533KB/s, mint=20295msec, maxt=20295msec

Disk stats (read/write):
  sdb: ios=10168/0, merge=0/0, ticks=29176/0, in_queue=29140, util=96.41%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [258M/0K /s] [252/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2646
  read : io=1024MB, bw=257686KB/s, iops=251, runt= 20346msec
    clat (usec): min=3850, max=6648, avg=3965.75, stdev=75.09
    bw (KB/s) : min=256508, max=258544, per=100.09%, avg=257929.10, stdev=407.53
  cpu          : usr=0.37%, sys=5.21%, ctx=5253, majf=0, minf=283
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=5120/0, short=0/0

     lat (msec): 4=97.27%, 10=2.73%

Run status group 0 (all jobs):
   READ: io=5120MB, aggrb=257686KB/s, minb=263870KB/s, maxb=263870KB/s, mint=20346msec, maxt=20346msec

Disk stats (read/write):
  sdb: ios=10144/0, merge=0/0, ticks=28540/0, in_queue=28488, util=94.62%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [261M/0K /s] [255/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2653
  read : io=2991MB, bw=260744KB/s, iops=254, runt= 60004msec
    clat (usec): min=3826, max=6575, avg=3919.83, stdev=53.34
    bw (KB/s) : min=256508, max=262144, per=100.10%, avg=261009.11, stdev=1336.16
  cpu          : usr=0.24%, sys=4.85%, ctx=15807, majf=0, minf=298
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15279/0, short=0/0

     lat (msec): 4=98.63%, 10=1.37%

Run status group 0 (all jobs):
   READ: io=15279MB, aggrb=260744KB/s, minb=267002KB/s, maxb=267002KB/s, mint=60004msec, maxt=60004msec

Disk stats (read/write):
  sdb: ios=30516/0, merge=0/0, ticks=90136/0, in_queue=90080, util=98.31%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing read tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=4k --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [70836K/0K /s] [17K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2657
  read : io=4036MB, bw=68886KB/s, iops=17221, runt= 60001msec
    clat (usec): min=61, max=3873, avg=228.52, stdev=20.06
    bw (KB/s) : min=16056, max=19096, per=24.99%, avg=17217.21, stdev=171.71
  cpu          : usr=2.22%, sys=6.31%, ctx=1037289, majf=0, minf=131
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=1033303/0, short=0/0
     lat (usec): 100=0.22%, 250=90.98%, 500=8.70%, 750=0.03%, 1000=0.02%
     lat (msec): 2=0.05%, 4=0.01%

Run status group 0 (all jobs):
   READ: io=4036MB, aggrb=68885KB/s, minb=70538KB/s, maxb=70538KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=1031882/0, merge=0/0, ticks=212008/0, in_queue=211568, util=99.87%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=4k --size=16004188569 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [68382K/0K /s] [17K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2669
  read : io=20796KB, bw=70250KB/s, iops=17562, runt= 60001msec
    clat (usec): min=55, max=3768, avg=224.34, stdev=20.80
    bw (KB/s) : min=15912, max=19672, per=24.99%, avg=17552.43, stdev=353.23
  cpu          : usr=2.04%, sys=5.99%, ctx=1058542, majf=0, minf=131
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=1053775/0, short=0/0
     lat (usec): 100=0.27%, 250=95.21%, 500=4.44%, 750=0.01%, 1000=0.02%
     lat (msec): 2=0.05%, 4=0.01%

Run status group 0 (all jobs):
   READ: io=4116MB, aggrb=70250KB/s, minb=71936KB/s, maxb=71936KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=1052494/0, merge=0/0, ticks=210992/0, in_queue=210588, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [87846K/0K /s] [21K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2677
  read : io=970036KB, bw=86071KB/s, iops=21517, runt= 60001msec
    clat (usec): min=43, max=4963, avg=178.12, stdev=53.39
    bw (KB/s) : min=18320, max=25208, per=25.18%, avg=21672.06, stdev=825.38
  cpu          : usr=3.07%, sys=12.21%, ctx=1478911, majf=0, minf=9663
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=1291085/0, short=0/0
     lat (usec): 50=0.01%, 100=12.97%, 250=68.25%, 500=17.12%, 750=1.48%
     lat (usec): 1000=0.13%
     lat (msec): 2=0.05%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=5043MB, aggrb=86070KB/s, minb=88136KB/s, maxb=88136KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=1288534/0, merge=0/0, ticks=192396/0, in_queue=190748, util=99.67%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --size=16004188569 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [86884K/0K /s] [21K/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2683
  read : io=914636KB, bw=85148KB/s, iops=21286, runt= 60001msec
    clat (usec): min=40, max=13854, avg=181.36, stdev=53.60
    bw (KB/s) : min=18432, max=24808, per=24.99%, avg=21279.58, stdev=653.16
  cpu          : usr=3.00%, sys=11.79%, ctx=1461857, majf=0, minf=1559
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=1277235/0, short=0/0
     lat (usec): 50=0.01%, 100=11.82%, 250=68.69%, 500=17.79%, 750=1.52%
     lat (usec): 1000=0.12%
     lat (msec): 2=0.04%, 4=0.01%, 10=0.01%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=4989MB, aggrb=85147KB/s, minb=87191KB/s, maxb=87191KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=1274884/0, merge=0/0, ticks=191088/0, in_queue=189348, util=99.77%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [264M/0K /s] [258/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2693
  read : io=3078MB, bw=262172KB/s, iops=256, runt= 60017msec
    clat (msec): min=3, max=504, avg=15.64, stdev=29.19
    bw (KB/s) : min= 3351, max=146576, per=26.84%, avg=70359.50, stdev=15121.98
  cpu          : usr=0.05%, sys=1.31%, ctx=15829, majf=0, minf=1153
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15366/0, short=0/0

     lat (msec): 4=94.36%, 10=1.41%, 20=0.34%, 50=0.02%, 100=0.01%
     lat (msec): 250=0.10%, 500=3.76%, 750=0.01%

Run status group 0 (all jobs):
   READ: io=15366MB, aggrb=262172KB/s, minb=268464KB/s, maxb=268464KB/s, mint=60017msec, maxt=60017msec

Disk stats (read/write):
  sdb: ios=30685/0, merge=0/0, ticks=424556/0, in_queue=425260, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [263M/0K /s] [257/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2699
  read : io=3082MB, bw=262258KB/s, iops=256, runt= 60013msec
    clat (msec): min=3, max=418, avg=15.57, stdev=29.25
    bw (KB/s) : min= 3271, max=112640, per=27.02%, avg=70874.66, stdev=15338.11
  cpu          : usr=0.05%, sys=1.18%, ctx=15774, majf=0, minf=1147
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15370/0, short=0/0

     lat (msec): 4=94.32%, 10=1.65%, 20=0.14%, 50=0.03%, 250=0.08%
     lat (msec): 500=3.79%

Run status group 0 (all jobs):
   READ: io=15370MB, aggrb=262257KB/s, minb=268552KB/s, maxb=268552KB/s, mint=60013msec, maxt=60013msec

Disk stats (read/write):
  sdb: ios=30693/0, merge=0/0, ticks=428940/0, in_queue=429668, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [261M/0K /s] [255/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2705
  read : io=3034MB, bw=261417KB/s, iops=255, runt= 60018msec
    clat (msec): min=3, max=509, avg=15.73, stdev=29.42
    bw (KB/s) : min= 3292, max=105382, per=26.61%, avg=69568.94, stdev=14660.05
  cpu          : usr=0.03%, sys=1.17%, ctx=15676, majf=0, minf=1151
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15322/0, short=0/0

     lat (msec): 4=94.10%, 10=2.02%, 50=0.01%, 250=0.06%, 500=3.81%
     lat (msec): 750=0.01%

Run status group 0 (all jobs):
   READ: io=15322MB, aggrb=261417KB/s, minb=267691KB/s, maxb=267691KB/s, mint=60018msec, maxt=60018msec

Disk stats (read/write):
  sdb: ios=30598/0, merge=0/0, ticks=420396/0, in_queue=421000, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [264M/0K /s] [258/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2711
  read : io=3324MB, bw=266387KB/s, iops=260, runt= 60013msec
    clat (msec): min=3, max=411, avg=15.62, stdev=23.29
    bw (KB/s) : min=12300, max=147456, per=25.32%, avg=67440.17, stdev=10856.41
  cpu          : usr=0.03%, sys=1.12%, ctx=15927, majf=0, minf=1151
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15612/0, short=0/0

     lat (msec): 4=61.61%, 10=5.25%, 20=25.45%, 50=4.79%, 100=0.24%
     lat (msec): 250=0.71%, 500=1.94%

Run status group 0 (all jobs):
   READ: io=15612MB, aggrb=266387KB/s, minb=272780KB/s, maxb=272780KB/s, mint=60013msec, maxt=60013msec

Disk stats (read/write):
  sdb: ios=31184/0, merge=0/0, ticks=411504/0, in_queue=412644, util=99.90%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [262M/0K /s] [256/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2723
  read : io=3369MB, bw=267159KB/s, iops=260, runt= 60012msec
    clat (msec): min=3, max=429, avg=15.36, stdev=22.34
    bw (KB/s) : min= 8917, max=126722, per=25.50%, avg=68119.11, stdev=8226.11
  cpu          : usr=0.03%, sys=1.04%, ctx=16064, majf=0, minf=1151
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15657/0, short=0/0

     lat (msec): 4=56.75%, 10=5.83%, 20=29.35%, 50=5.36%, 100=0.27%
     lat (msec): 250=0.63%, 500=1.81%

Run status group 0 (all jobs):
   READ: io=15657MB, aggrb=267159KB/s, minb=273571KB/s, maxb=273571KB/s, mint=60012msec, maxt=60012msec

Disk stats (read/write):
  sdb: ios=31268/0, merge=0/0, ticks=414688/0, in_queue=414580, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [274M/0K /s] [268/0 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2729
  read : io=3401MB, bw=267692KB/s, iops=261, runt= 60015msec
    clat (msec): min=3, max=418, avg=15.48, stdev=21.32
    bw (KB/s) : min=17564, max=128766, per=25.29%, avg=67698.99, stdev=8499.28
  cpu          : usr=0.05%, sys=1.08%, ctx=16065, majf=0, minf=1216
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=15689/0, short=0/0

     lat (msec): 4=52.86%, 10=6.03%, 20=32.78%, 50=5.73%, 100=0.22%
     lat (msec): 250=0.86%, 500=1.52%

Run status group 0 (all jobs):
   READ: io=15689MB, aggrb=267692KB/s, minb=274116KB/s, maxb=274116KB/s, mint=60015msec, maxt=60015msec

Disk stats (read/write):
  sdb: ios=31332/0, merge=0/0, ticks=416608/0, in_queue=417416, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=4k --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/3764K /s] [0/919 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2740
  write: io=223280KB, bw=3721KB/s, iops=930, runt= 60002msec
    clat (usec): min=431, max=3845, avg=1070.36, stdev=534.41
    bw (KB/s) : min= 3280, max= 4720, per=100.14%, avg=3726.21, stdev=247.78
  cpu          : usr=0.54%, sys=2.51%, ctx=56114, majf=0, minf=31
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/55820, short=0/0
     lat (usec): 500=3.79%, 750=47.47%, 1000=0.23%
     lat (msec): 2=48.35%, 4=0.16%

Run status group 0 (all jobs):
  WRITE: io=223280KB, aggrb=3721KB/s, minb=3810KB/s, maxb=3810KB/s, mint=60002msec, maxt=60002msec

Disk stats (read/write):
  sdb: ios=19/55771, merge=0/0, ticks=0/55776, in_queue=55744, util=92.92%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=4k --size=16004188569 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/3772K /s] [0/921 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2762
  write: io=226380KB, bw=3773KB/s, iops=943, runt= 60001msec
    clat (usec): min=431, max=3315, avg=1055.72, stdev=537.34
    bw (KB/s) : min= 3336, max= 4702, per=100.06%, avg=3774.30, stdev=240.68
  cpu          : usr=0.45%, sys=2.08%, ctx=56867, majf=0, minf=31
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/56595, short=0/0
     lat (usec): 500=6.20%, 750=45.13%, 1000=0.19%
     lat (msec): 2=48.33%, 4=0.14%

Run status group 0 (all jobs):
  WRITE: io=226380KB, aggrb=3772KB/s, minb=3863KB/s, maxb=3863KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=26/56492, merge=0/0, ticks=4/57240, in_queue=57204, util=95.35%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/4407K /s] [0/1076 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2779
  write: io=232116KB, bw=3869KB/s, iops=967, runt= 60001msec
    clat (usec): min=432, max=3991, avg=1027.03, stdev=539.06
    bw (KB/s) : min= 3304, max= 4760, per=100.10%, avg=3871.70, stdev=245.30
  cpu          : usr=0.59%, sys=2.10%, ctx=58232, majf=0, minf=3580
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/58029, short=0/0
     lat (usec): 500=12.51%, 750=39.09%, 1000=0.04%
     lat (msec): 2=48.32%, 4=0.03%

Run status group 0 (all jobs):
  WRITE: io=232116KB, aggrb=3868KB/s, minb=3961KB/s, maxb=3961KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdb: ios=26/57938, merge=0/0, ticks=4/58700, in_queue=58676, util=97.80%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --size=16004188569 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/3813K /s] [0/931 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2796
  write: io=223008KB, bw=3717KB/s, iops=929, runt= 60002msec
    clat (usec): min=444, max=4893, avg=1070.20, stdev=541.44
    bw (KB/s) : min= 3336, max= 4686, per=100.16%, avg=3721.81, stdev=252.77
  cpu          : usr=0.51%, sys=1.90%, ctx=55938, majf=0, minf=387
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/55752, short=0/0
     lat (usec): 500=4.43%, 750=46.49%, 1000=0.51%
     lat (msec): 2=48.14%, 4=0.42%, 10=0.01%

Run status group 0 (all jobs):
  WRITE: io=223008KB, aggrb=3716KB/s, minb=3805KB/s, maxb=3805KB/s, mint=60002msec, maxt=60002msec

Disk stats (read/write):
  sdb: ios=25/55695, merge=0/0, ticks=4/56144, in_queue=56112, util=93.53%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/88080K /s] [0/84 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2813
  write: io=947200KB, bw=85686KB/s, iops=83, runt= 60004msec
    clat (msec): min=8, max=284, avg=11.95, stdev= 6.78
    bw (KB/s) : min=41918, max=89752, per=100.40%, avg=86031.73, stdev=6601.69
  cpu          : usr=0.08%, sys=0.91%, ctx=5153, majf=0, minf=31
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5021, short=0/0

     lat (msec): 10=4.98%, 20=94.80%, 50=0.16%, 500=0.06%

Run status group 0 (all jobs):
  WRITE: io=5021MB, aggrb=85686KB/s, minb=87742KB/s, maxb=87742KB/s, mint=60004msec, maxt=60004msec

Disk stats (read/write):
  sdb: ios=8/10027, merge=0/0, ticks=4/88844, in_queue=88820, util=99.03%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/99614K /s] [0/95 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2831
  write: io=1024MB, bw=92390KB/s, iops=90, runt= 56747msec
    clat (msec): min=7, max=284, avg=11.08, stdev= 5.54
    bw (KB/s) : min=36426, max=98304, per=100.09%, avg=92477.64, stdev=8535.30
  cpu          : usr=0.08%, sys=1.18%, ctx=5276, majf=0, minf=30
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0

     lat (msec): 10=24.14%, 20=75.78%, 50=0.04%, 500=0.04%

Run status group 0 (all jobs):
  WRITE: io=5120MB, aggrb=92390KB/s, minb=94607KB/s, maxb=94607KB/s, mint=56747msec, maxt=56747msec

Disk stats (read/write):
  sdb: ios=15/10225, merge=0/0, ticks=4/84180, in_queue=84172, util=98.93%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/99M /s] [0/97 iops] [eta 00m:00s]  
file1: (groupid=0, jobs=1): err= 0: pid=2851
  write: io=1174MB, bw=89925KB/s, iops=87, runt= 60011msec
    clat (msec): min=5, max=306, avg=11.38, stdev= 6.89
    bw (KB/s) : min=38834, max=99162, per=100.30%, avg=90193.47, stdev=8294.03
  cpu          : usr=0.05%, sys=1.07%, ctx=5415, majf=0, minf=30
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5270, short=0/0

     lat (msec): 10=24.19%, 20=75.69%, 50=0.06%, 500=0.06%

Run status group 0 (all jobs):
  WRITE: io=5270MB, aggrb=89924KB/s, minb=92083KB/s, maxb=92083KB/s, mint=60011msec, maxt=60011msec

Disk stats (read/write):
  sdb: ios=0/10523, merge=0/0, ticks=0/89060, in_queue=89048, util=99.30%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/78643K /s] [0/75 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2867
  write: io=1024MB, bw=89006KB/s, iops=86, runt= 58905msec
    clat (msec): min=5, max=283, avg=11.50, stdev= 5.65
    bw (KB/s) : min=36426, max=99358, per=100.20%, avg=89180.84, stdev=9099.80
  cpu          : usr=0.04%, sys=0.96%, ctx=5245, majf=0, minf=29
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0

     lat (msec): 10=19.51%, 20=80.29%, 50=0.16%, 500=0.04%

Run status group 0 (all jobs):
  WRITE: io=5120MB, aggrb=89005KB/s, minb=91141KB/s, maxb=91141KB/s, mint=58905msec, maxt=58905msec

Disk stats (read/write):
  sdb: ios=0/10206, merge=0/0, ticks=0/87672, in_queue=87640, util=98.84%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/77594K /s] [0/74 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2887
  write: io=1024MB, bw=89143KB/s, iops=87, runt= 58814msec
    clat (msec): min=5, max=283, avg=11.48, stdev= 6.81
    bw (KB/s) : min=36283, max=98107, per=100.16%, avg=89281.87, stdev=10204.56
  cpu          : usr=0.03%, sys=1.26%, ctx=5284, majf=0, minf=29
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5120, short=0/0

     lat (msec): 10=17.50%, 20=82.30%, 50=0.14%, 500=0.06%

Run status group 0 (all jobs):
  WRITE: io=5120MB, aggrb=89143KB/s, minb=91282KB/s, maxb=91282KB/s, mint=58814msec, maxt=58814msec

Disk stats (read/write):
  sdb: ios=15/10218, merge=0/0, ticks=4/87532, in_queue=87476, util=98.76%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=160041885696 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/89128K /s] [0/85 iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=2904
  write: io=1274MB, bw=91640KB/s, iops=89, runt= 60005msec
    clat (msec): min=5, max=284, avg=11.17, stdev= 6.58
    bw (KB/s) : min=40474, max=100151, per=100.39%, avg=92001.19, stdev=8525.13
  cpu          : usr=0.05%, sys=1.11%, ctx=5521, majf=0, minf=42
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5370, short=0/0

     lat (msec): 10=21.62%, 20=78.27%, 50=0.06%, 500=0.06%

Run status group 0 (all jobs):
  WRITE: io=5370MB, aggrb=91640KB/s, minb=93839KB/s, maxb=93839KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdb: ios=7/10726, merge=0/0, ticks=4/89344, in_queue=89320, util=99.36%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=4k --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/4034K /s] [0/985 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2922
  write: io=223696KB, bw=3728KB/s, iops=932, runt= 60003msec
    clat (usec): min=435, max=396314, avg=4337.18, stdev=15458.80
    bw (KB/s) : min=  172, max= 2315, per=24.96%, avg=930.63, stdev=126.55
  cpu          : usr=0.11%, sys=0.71%, ctx=56232, majf=0, minf=128
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/55924, short=0/0
     lat (usec): 500=4.27%, 750=46.63%, 1000=0.21%
     lat (msec): 2=47.52%, 4=0.24%, 10=0.02%, 20=0.01%, 50=0.01%
     lat (msec): 100=0.02%, 250=0.04%, 500=1.04%

Run status group 0 (all jobs):
  WRITE: io=223696KB, aggrb=3728KB/s, minb=3817KB/s, maxb=3817KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdb: ios=39/55841, merge=0/0, ticks=212/229152, in_queue=229708, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=4k --size=16004188569 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/4296K /s] [0/1049 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2947
  write: io=226156KB, bw=3769KB/s, iops=942, runt= 60008msec
    clat (usec): min=429, max=394374, avg=4307.18, stdev=15476.59
    bw (KB/s) : min=   22, max= 2336, per=25.47%, avg=959.83, stdev=169.32
  cpu          : usr=0.13%, sys=0.64%, ctx=56827, majf=0, minf=127
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/56539, short=0/0
     lat (usec): 500=7.00%, 750=43.93%, 1000=0.17%
     lat (msec): 2=47.56%, 4=0.19%, 10=0.02%, 20=0.01%, 50=0.02%
     lat (msec): 100=0.05%, 250=0.07%, 500=0.98%

Run status group 0 (all jobs):
  WRITE: io=226156KB, aggrb=3768KB/s, minb=3859KB/s, maxb=3859KB/s, mint=60008msec, maxt=60008msec

Disk stats (read/write):
  sdb: ios=26/56430, merge=0/0, ticks=8/231980, in_queue=232460, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/3883K /s] [0/948 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2970
  write: io=227756KB, bw=3795KB/s, iops=948, runt= 60012msec
    clat (usec): min=426, max=409928, avg=4373.88, stdev=14199.76
    bw (KB/s) : min=   31, max= 2283, per=24.97%, avg=947.55, stdev=218.15
  cpu          : usr=0.13%, sys=0.60%, ctx=57159, majf=0, minf=7561
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/56939, short=0/0
     lat (usec): 500=8.92%, 750=41.63%, 1000=0.19%
     lat (msec): 2=46.89%, 4=0.51%, 10=0.36%, 20=0.13%, 50=0.08%
     lat (msec): 100=0.05%, 250=0.60%, 500=0.64%

Run status group 0 (all jobs):
  WRITE: io=227756KB, aggrb=3795KB/s, minb=3886KB/s, maxb=3886KB/s, mint=60012msec, maxt=60012msec

Disk stats (read/write):
  sdb: ios=51/56937, merge=0/0, ticks=8/235948, in_queue=235944, util=99.85%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --size=16004188569 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/3796K /s] [0/927 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=2997
  write: io=224040KB, bw=3732KB/s, iops=932, runt= 60036msec
    clat (usec): min=431, max=417800, avg=4240.14, stdev=14177.66
    bw (KB/s) : min=   25, max= 2656, per=25.95%, avg=968.35, stdev=245.91
  cpu          : usr=0.14%, sys=0.58%, ctx=56204, majf=0, minf=1191
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/56010, short=0/0
     lat (usec): 500=5.39%, 750=44.19%, 1000=1.00%
     lat (msec): 2=46.72%, 4=0.84%, 10=0.35%, 20=0.10%, 50=0.07%
     lat (msec): 100=0.06%, 250=0.62%, 500=0.65%

Run status group 0 (all jobs):
  WRITE: io=224040KB, aggrb=3731KB/s, minb=3821KB/s, maxb=3821KB/s, mint=60036msec, maxt=60036msec

Disk stats (read/write):
  sdb: ios=26/55880, merge=0/0, ticks=4/235468, in_queue=235872, util=99.87%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/98566K /s] [0/94 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=3022
  write: io=1417MB, bw=94037KB/s, iops=91, runt= 60033msec
    clat (msec): min=7, max=638, avg=43.60, stdev=49.24
    bw (KB/s) : min= 2904, max=40554, per=26.88%, avg=25279.07, stdev=4862.40
  cpu          : usr=0.02%, sys=0.32%, ctx=5622, majf=0, minf=127
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5513, short=0/0

     lat (msec): 10=30.44%, 20=58.30%, 50=1.16%, 100=0.04%, 250=0.16%
     lat (msec): 500=9.74%, 750=0.16%

Run status group 0 (all jobs):
  WRITE: io=5513MB, aggrb=94036KB/s, minb=96293KB/s, maxb=96293KB/s, mint=60033msec, maxt=60033msec

Disk stats (read/write):
  sdb: ios=13/11004, merge=0/0, ticks=4/432980, in_queue=434048, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/92274K /s] [0/88 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=3046
  write: io=1367MB, bw=93189KB/s, iops=91, runt= 60030msec
    clat (msec): min=5, max=674, avg=43.58, stdev=49.44
    bw (KB/s) : min= 1796, max=40474, per=27.10%, avg=25254.57, stdev=4827.53
  cpu          : usr=0.01%, sys=0.25%, ctx=5538, majf=0, minf=123
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5463, short=0/0

     lat (msec): 10=26.63%, 20=61.54%, 50=1.45%, 100=0.24%, 250=0.33%
     lat (msec): 500=9.63%, 750=0.18%

Run status group 0 (all jobs):
  WRITE: io=5463MB, aggrb=93188KB/s, minb=95425KB/s, maxb=95425KB/s, mint=60030msec, maxt=60030msec

Disk stats (read/write):
  sdb: ios=17/10906, merge=0/0, ticks=8/438776, in_queue=439548, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/87031K /s] [0/83 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=3066
  write: io=1145MB, bw=89275KB/s, iops=87, runt= 60115msec
    clat (msec): min=6, max=649, avg=45.90, stdev=49.15
    bw (KB/s) : min= 3757, max=50615, per=26.66%, avg=23804.02, stdev=4503.06
  cpu          : usr=0.01%, sys=0.27%, ctx=5323, majf=0, minf=123
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5241, short=0/0

     lat (msec): 10=19.92%, 20=67.49%, 50=1.68%, 100=0.08%, 250=0.31%
     lat (msec): 500=10.36%, 750=0.17%

Run status group 0 (all jobs):
  WRITE: io=5241MB, aggrb=89275KB/s, minb=91417KB/s, maxb=91417KB/s, mint=60115msec, maxt=60115msec

Disk stats (read/write):
  sdb: ios=10/10470, merge=0/0, ticks=0/414108, in_queue=416072, util=99.87%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/89128K /s] [0/85 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=3089
  write: io=1298MB, bw=92015KB/s, iops=89, runt= 60028msec
    clat (msec): min=6, max=427, avg=44.61, stdev=24.81
    bw (KB/s) : min= 7460, max=46271, per=25.16%, avg=23153.19, stdev=2176.02
  cpu          : usr=0.02%, sys=0.24%, ctx=5474, majf=0, minf=135
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5394, short=0/0

     lat (msec): 10=2.17%, 20=17.19%, 50=58.16%, 100=19.61%, 250=0.87%
     lat (msec): 500=2.00%

Run status group 0 (all jobs):
  WRITE: io=5394MB, aggrb=92014KB/s, minb=94223KB/s, maxb=94223KB/s, mint=60028msec, maxt=60028msec

Disk stats (read/write):
  sdb: ios=0/10768, merge=0/0, ticks=0/438048, in_queue=438876, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=154673176576 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/98M /s] [0/96 iops] [eta 00m:00s]  
file1: (groupid=0, jobs=4): err= 0: pid=3106
  write: io=1276MB, bw=91626KB/s, iops=89, runt= 60037msec
    clat (msec): min=5, max=671, avg=45.49, stdev=22.71
    bw (KB/s) : min= 1526, max=55197, per=24.81%, avg=22735.95, stdev=2511.76
  cpu          : usr=0.02%, sys=0.25%, ctx=5457, majf=0, minf=132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5372, short=0/0

     lat (msec): 10=3.72%, 20=12.06%, 50=60.55%, 100=20.85%, 250=1.38%
     lat (msec): 500=1.38%, 750=0.06%

Run status group 0 (all jobs):
  WRITE: io=5372MB, aggrb=91625KB/s, minb=93824KB/s, maxb=93824KB/s, mint=60037msec, maxt=60037msec

Disk stats (read/write):
  sdb: ios=11/10721, merge=0/0, ticks=0/439248, in_queue=439368, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/local/bin/fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=160041885696 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/99614K /s] [0/95 iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=3127
  write: io=1245MB, bw=91094KB/s, iops=88, runt= 60039msec
    clat (msec): min=6, max=507, avg=44.26, stdev=29.16
    bw (KB/s) : min= 4087, max=48956, per=25.77%, avg=23477.61, stdev=2617.31
  cpu          : usr=0.01%, sys=0.26%, ctx=5410, majf=0, minf=165
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w: total=0/5341, short=0/0

     lat (msec): 10=6.38%, 20=22.43%, 50=50.18%, 100=16.98%, 250=1.24%
     lat (msec): 500=2.77%, 750=0.02%

Run status group 0 (all jobs):
  WRITE: io=5341MB, aggrb=91093KB/s, minb=93280KB/s, maxb=93280KB/s, mint=60039msec, maxt=60039msec

Disk stats (read/write):
  sdb: ios=18/10660, merge=0/0, ticks=4/433964, in_queue=434044, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
wfischer@wfischer-t410-ubuntu:~$