[root@test fiotest]# sh fio.sh 
fio 1.54

/dev/sdc:
 HDIO_DRIVE_CMD(identify) failed: Inappropriate ioctl for device
#################################################################################
Executing read tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [93452K/0K /s] [22.9K/0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=5930
  read : io=5403.7MB, bw=92221KB/s, iops=23055 , runt= 60001msec
    clat (usec): min=29 , max=4970 , avg=41.13, stdev=19.95
     lat (usec): min=30 , max=4971 , avg=41.39, stdev=20.03
    bw (KB/s) : min=91112, max=95016, per=100.01%, avg=92225.21, stdev=870.99
  cpu          : usr=6.61%, sys=24.96%, ctx=1375662, majf=0, minf=30
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=1383337/0/0, short=0/0/0
     lat (usec): 50=95.19%, 100=0.73%, 250=4.06%, 500=0.01%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=5403.7MB, aggrb=92220KB/s, minb=94434KB/s, maxb=94434KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=1380533/0, merge=0/0, ticks=48327/0, in_queue=48311, util=80.86%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [93681K/0K /s] [22.9K/0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=5952
  read : io=5383.8MB, bw=91881KB/s, iops=22970 , runt= 60001msec
    clat (usec): min=29 , max=3979 , avg=41.31, stdev=29.74
     lat (usec): min=30 , max=3979 , avg=41.57, stdev=29.80
    bw (KB/s) : min=61936, max=96112, per=100.01%, avg=91890.22, stdev=3006.15
  cpu          : usr=6.49%, sys=25.36%, ctx=1370466, majf=0, minf=30
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=1378245/0/0, short=0/0/0
     lat (usec): 50=95.18%, 100=0.73%, 250=4.08%, 500=0.01%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%

Run status group 0 (all jobs):
   READ: io=5383.8MB, aggrb=91881KB/s, minb=94086KB/s, maxb=94086KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=1375431/0, merge=0/0, ticks=48338/0, in_queue=48313, util=80.84%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [27279K/0K /s] [6660 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6026
  read : io=1564.4MB, bw=26698KB/s, iops=6674 , runt= 60001msec
    clat (usec): min=36 , max=9134 , avg=145.90, stdev=62.41
     lat (usec): min=36 , max=9134 , avg=146.22, stdev=62.48
    bw (KB/s) : min=23904, max=27232, per=100.02%, avg=26702.18, stdev=343.02
  cpu          : usr=4.73%, sys=16.70%, ctx=395467, majf=0, minf=1060
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=400475/0/0, short=0/0/0
     lat (usec): 50=0.09%, 100=0.10%, 250=94.70%, 500=5.06%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.04%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=1564.4MB, aggrb=26697KB/s, minb=27338KB/s, maxb=27338KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=399580/0, merge=0/0, ticks=53056/0, in_queue=53028, util=88.73%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [27504K/0K /s] [6714 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6048
  read : io=1575.3MB, bw=26885KB/s, iops=6721 , runt= 60000msec
    clat (usec): min=37 , max=7675 , avg=144.95, stdev=47.93
     lat (usec): min=37 , max=7676 , avg=145.28, stdev=48.04
    bw (KB/s) : min=24264, max=27272, per=100.02%, avg=26889.61, stdev=331.34
  cpu          : usr=4.55%, sys=16.63%, ctx=398168, majf=0, minf=1060
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=403270/0/0, short=0/0/0
     lat (usec): 50=0.16%, 100=0.12%, 250=94.72%, 500=4.97%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=1575.3MB, aggrb=26884KB/s, minb=27529KB/s, maxb=27529KB/s, mint=60000msec, maxt=60000msec

Disk stats (read/write):
  sdc: ios=402373/0, merge=0/0, ticks=53477/0, in_queue=53470, util=89.46%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [270.6M/0K /s] [264 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6093
  read : io=5120.0MB, bw=270238KB/s, iops=263 , runt= 19401msec
    clat (usec): min=783 , max=10984 , avg=3785.96, stdev=164.10
     lat (usec): min=784 , max=10984 , avg=3786.23, stdev=164.08
    bw (KB/s) : min=261620, max=270759, per=100.10%, avg=270507.37, stdev=1482.27
  cpu          : usr=0.11%, sys=2.76%, ctx=5126, majf=0, minf=283
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=5120/0/0, short=0/0/0
     lat (usec): 1000=0.06%
     lat (msec): 2=0.02%, 4=99.16%, 10=0.74%, 20=0.02%

Run status group 0 (all jobs):
   READ: io=5120.0MB, aggrb=270237KB/s, minb=276723KB/s, maxb=276723KB/s, mint=19401msec, maxt=19401msec

Disk stats (read/write):
  sdc: ios=20289/0, merge=0/0, ticks=55509/0, in_queue=55518, util=98.44%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [270.6M/0K /s] [264 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6111
  read : io=5120.0MB, bw=270391KB/s, iops=264 , runt= 19390msec
    clat (usec): min=820 , max=11701 , avg=3783.55, stdev=195.06
     lat (usec): min=820 , max=11701 , avg=3783.84, stdev=195.06
    bw (KB/s) : min=267752, max=270759, per=100.10%, avg=270668.74, stdev=490.76
  cpu          : usr=0.10%, sys=2.66%, ctx=5124, majf=0, minf=283
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=5120/0/0, short=0/0/0
     lat (usec): 1000=0.10%
     lat (msec): 4=99.61%, 10=0.27%, 20=0.02%

Run status group 0 (all jobs):
   READ: io=5120.0MB, aggrb=270390KB/s, minb=276880KB/s, maxb=276880KB/s, mint=19390msec, maxt=19390msec

Disk stats (read/write):
  sdc: ios=20303/0, merge=0/0, ticks=25642/0, in_queue=25646, util=98.45%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [R] [100.0% done] [270.6M/0K /s] [264 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6115
  read : io=15847MB, bw=270455KB/s, iops=264 , runt= 60000msec
    clat (usec): min=786 , max=16517 , avg=3782.64, stdev=152.96
     lat (usec): min=786 , max=16517 , avg=3782.94, stdev=152.94
    bw (KB/s) : min=267752, max=271298, per=100.10%, avg=270729.25, stdev=296.54
  cpu          : usr=0.13%, sys=2.70%, ctx=15858, majf=0, minf=285
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=15847/0/0, short=0/0/0
     lat (usec): 1000=0.04%
     lat (msec): 2=0.01%, 4=99.79%, 10=0.15%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=15847MB, aggrb=270455KB/s, minb=276946KB/s, maxb=276946KB/s, mint=60000msec, maxt=60000msec

Disk stats (read/write):
  sdc: ios=63259/0, merge=0/0, ticks=79791/0, in_queue=79794, util=98.82%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [245.3M/0K /s] [239 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6204
  read : io=5120.0MB, bw=244629KB/s, iops=238 , runt= 21432msec
    clat (usec): min=2433 , max=9813 , avg=4180.09, stdev=141.98
     lat (usec): min=2434 , max=9813 , avg=4180.43, stdev=141.99
    bw (KB/s) : min=240701, max=245760, per=100.10%, avg=244876.71, stdev=780.57
  cpu          : usr=0.23%, sys=2.82%, ctx=5125, majf=0, minf=282
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=5120/0/0, short=0/0/0

     lat (msec): 4=0.14%, 10=99.86%

Run status group 0 (all jobs):
   READ: io=5120.0MB, aggrb=244628KB/s, minb=250499KB/s, maxb=250499KB/s, mint=21432msec, maxt=21432msec

Disk stats (read/write):
  sdc: ios=20286/0, merge=0/0, ticks=54010/0, in_queue=54017, util=98.02%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [245.3M/0K /s] [239 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6218
  read : io=5120.0MB, bw=244583KB/s, iops=238 , runt= 21436msec
    clat (usec): min=2380 , max=7006 , avg=4180.90, stdev=130.14
     lat (usec): min=2380 , max=7006 , avg=4181.22, stdev=130.15
    bw (KB/s) : min=241181, max=245760, per=100.11%, avg=244841.90, stdev=772.48
  cpu          : usr=0.21%, sys=2.90%, ctx=5123, majf=0, minf=281
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=5120/0/0, short=0/0/0

     lat (msec): 4=0.14%, 10=99.86%

Run status group 0 (all jobs):
   READ: io=5120.0MB, aggrb=244582KB/s, minb=250452KB/s, maxb=250452KB/s, mint=21436msec, maxt=21436msec

Disk stats (read/write):
  sdc: ios=20283/0, merge=0/0, ticks=54008/0, in_queue=54012, util=98.01%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [249.2M/0K /s] [243 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6225
  read : io=14569MB, bw=248632KB/s, iops=242 , runt= 60003msec
    clat (usec): min=739 , max=13234 , avg=4113.41, stdev=188.37
     lat (usec): min=740 , max=13234 , avg=4113.76, stdev=188.39
    bw (KB/s) : min=231424, max=251936, per=100.11%, avg=248898.30, stdev=1876.80
  cpu          : usr=0.13%, sys=2.70%, ctx=14581, majf=0, minf=295
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=14569/0/0, short=0/0/0
     lat (usec): 750=0.01%, 1000=0.14%
     lat (msec): 4=0.01%, 10=99.82%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=14569MB, aggrb=248631KB/s, minb=254598KB/s, maxb=254598KB/s, mint=60003msec, maxt=60003msec

Disk stats (read/write):
  sdc: ios=58155/0, merge=0/0, ticks=151816/0, in_queue=151820, util=99.23%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing read tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [182.8M/0K /s] [45.7K/0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6279
  read : io=10437MB, bw=178124KB/s, iops=44530 , runt= 60001msec
    clat (usec): min=30 , max=145815 , avg=87.76, stdev=228.39
     lat (usec): min=30 , max=145816 , avg=87.95, stdev=228.39
    bw (KB/s) : min=30936, max=46840, per=25.01%, avg=44550.63, stdev=1581.43
  cpu          : usr=2.94%, sys=11.65%, ctx=2656634, majf=0, minf=116
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=2671899/0/0, short=0/0/0
     lat (usec): 50=3.77%, 100=82.87%, 250=13.29%, 500=0.04%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
     lat (msec): 100=0.01%, 250=0.01%

Run status group 0 (all jobs):
   READ: io=10437MB, aggrb=178123KB/s, minb=182398KB/s, maxb=182398KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=2663436/0, merge=2909/0, ticks=172013/0, in_queue=171802, util=94.59%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [180.8M/0K /s] [45.2K/0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6297
  read : io=10540MB, bw=179879KB/s, iops=44969 , runt= 60001msec
    clat (usec): min=29 , max=96444 , avg=86.91, stdev=166.24
     lat (usec): min=29 , max=96444 , avg=87.11, stdev=166.24
    bw (KB/s) : min=34457, max=46776, per=25.01%, avg=44982.28, stdev=1135.63
  cpu          : usr=3.03%, sys=11.66%, ctx=2682688, majf=0, minf=124
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=2698226/0/0, short=0/0/0
     lat (usec): 50=3.83%, 100=82.61%, 250=13.49%, 500=0.04%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%
     lat (msec): 100=0.01%

Run status group 0 (all jobs):
   READ: io=10540MB, aggrb=179878KB/s, minb=184195KB/s, maxb=184195KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=2689745/0, merge=2713/0, ticks=170324/0, in_queue=170071, util=94.17%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [71424K/0K /s] [17.5K/0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6390
  read : io=4073.4MB, bw=69518KB/s, iops=17379 , runt= 60001msec
    clat (usec): min=39 , max=15197 , avg=226.70, stdev=64.12
     lat (usec): min=40 , max=15197 , avg=226.97, stdev=64.12
    bw (KB/s) : min=14603, max=18056, per=25.00%, avg=17376.90, stdev=234.94
  cpu          : usr=1.90%, sys=5.62%, ctx=1042807, majf=0, minf=4241
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=1042782/0/0, short=0/0/0
     lat (usec): 50=0.03%, 100=0.15%, 250=83.93%, 500=14.77%, 750=0.07%
     lat (usec): 1000=0.83%
     lat (msec): 2=0.04%, 4=0.18%, 10=0.01%, 20=0.01%

Run status group 0 (all jobs):
   READ: io=4073.4MB, aggrb=69517KB/s, minb=71186KB/s, maxb=71186KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=1040080/0, merge=0/0, ticks=228012/0, in_queue=227986, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [70848K/0K /s] [17.3K/0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6408
  read : io=4073.8MB, bw=69524KB/s, iops=17380 , runt= 60001msec
    clat (usec): min=38 , max=9740 , avg=226.59, stdev=62.57
     lat (usec): min=38 , max=9740 , avg=226.87, stdev=62.57
    bw (KB/s) : min=12896, max=18352, per=25.00%, avg=17383.12, stdev=283.82
  cpu          : usr=1.94%, sys=5.67%, ctx=1042959, majf=0, minf=7366
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=1042875/0/0, short=0/0/0
     lat (usec): 50=0.06%, 100=0.19%, 250=83.70%, 500=14.87%, 750=0.09%
     lat (usec): 1000=0.87%
     lat (msec): 2=0.05%, 4=0.15%, 10=0.01%

Run status group 0 (all jobs):
   READ: io=4073.8MB, aggrb=69523KB/s, minb=71192KB/s, maxb=71192KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=1040095/0, merge=0/0, ticks=227936/0, in_queue=227931, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [972.1M/0K /s] [950 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6449
  read : io=20480MB, bw=957735KB/s, iops=935 , runt= 21897msec
    clat (usec): min=721 , max=30583 , avg=4273.24, stdev=1134.29
     lat (usec): min=721 , max=30583 , avg=4273.48, stdev=1134.30
    bw (KB/s) : min=148612, max=271298, per=25.00%, avg=239453.30, stdev=10117.32
  cpu          : usr=0.11%, sys=2.22%, ctx=20503, majf=0, minf=1132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=20480/0/0, short=0/0/0
     lat (usec): 750=0.01%, 1000=0.76%
     lat (msec): 2=0.74%, 4=83.33%, 10=11.80%, 20=3.08%, 50=0.28%

Run status group 0 (all jobs):
   READ: io=20480MB, aggrb=957734KB/s, minb=980720KB/s, maxb=980720KB/s, mint=21897msec, maxt=21897msec

Disk stats (read/write):
  sdc: ios=81252/0, merge=0/0, ticks=290498/0, in_queue=290548, util=99.88%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [976.2M/0K /s] [953 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6472
  read : io=20480MB, bw=960366KB/s, iops=937 , runt= 21837msec
    clat (usec): min=721 , max=30584 , avg=4260.79, stdev=1110.95
     lat (usec): min=721 , max=30585 , avg=4261.06, stdev=1110.93
    bw (KB/s) : min=215793, max=270759, per=25.06%, avg=240621.23, stdev=6045.10
  cpu          : usr=0.08%, sys=2.13%, ctx=20511, majf=0, minf=1132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=20480/0/0, short=0/0/0
     lat (usec): 750=0.31%, 1000=0.81%
     lat (msec): 2=0.79%, 4=77.04%, 10=18.08%, 20=2.60%, 50=0.37%

Run status group 0 (all jobs):
   READ: io=20480MB, aggrb=960366KB/s, minb=983415KB/s, maxb=983415KB/s, mint=21837msec, maxt=21837msec

Disk stats (read/write):
  sdc: ios=81568/0, merge=0/0, ticks=264895/0, in_queue=264919, util=99.89%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [RRRR] [100.0% done] [979.8M/0K /s] [956 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6492
  read : io=55137MB, bw=940926KB/s, iops=918 , runt= 60005msec
    clat (usec): min=745 , max=30752 , avg=4349.54, stdev=1238.46
     lat (usec): min=745 , max=30752 , avg=4349.83, stdev=1238.46
    bw (KB/s) : min=195824, max=264192, per=25.02%, avg=235465.90, stdev=7440.33
  cpu          : usr=0.11%, sys=2.35%, ctx=55182, majf=0, minf=1144
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=55137/0/0, short=0/0/0
     lat (usec): 750=0.04%, 1000=1.23%
     lat (msec): 2=1.01%, 4=76.75%, 10=17.31%, 20=3.16%, 50=0.50%

Run status group 0 (all jobs):
   READ: io=55137MB, aggrb=940926KB/s, minb=963508KB/s, maxb=963508KB/s, mint=60005msec, maxt=60005msec

Disk stats (read/write):
  sdc: ios=220037/0, merge=0/0, ticks=744945/0, in_queue=744987, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [264.5M/0K /s] [258 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6576
  read : io=15496MB, bw=264412KB/s, iops=258 , runt= 60012msec
    clat (usec): min=889 , max=29670 , avg=15485.01, stdev=213.15
     lat (usec): min=890 , max=29670 , avg=15485.36, stdev=213.14
    bw (KB/s) : min=65145, max=70136, per=25.02%, avg=66153.60, stdev=154.65
  cpu          : usr=0.04%, sys=0.61%, ctx=15511, majf=0, minf=1132
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=15496/0/0, short=0/0/0
     lat (usec): 1000=0.05%
     lat (msec): 2=0.01%, 10=0.02%, 20=99.90%, 50=0.03%

Run status group 0 (all jobs):
   READ: io=15496MB, aggrb=264412KB/s, minb=270758KB/s, maxb=270758KB/s, mint=60012msec, maxt=60012msec

Disk stats (read/write):
  sdc: ios=61843/0, merge=0/0, ticks=863989/0, in_queue=864101, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [264.5M/0K /s] [258 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6585
  read : io=15488MB, bw=264267KB/s, iops=258 , runt= 60014msec
    clat (usec): min=865 , max=23030 , avg=15494.28, stdev=198.84
     lat (usec): min=865 , max=23030 , avg=15494.59, stdev=198.84
    bw (KB/s) : min=65145, max=70136, per=25.03%, avg=66147.17, stdev=139.86
  cpu          : usr=0.04%, sys=0.59%, ctx=15515, majf=0, minf=1140
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=15488/0/0, short=0/0/0
     lat (usec): 1000=0.06%
     lat (msec): 10=0.02%, 20=99.92%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=15488MB, aggrb=264266KB/s, minb=270609KB/s, maxb=270609KB/s, mint=60014msec, maxt=60014msec

Disk stats (read/write):
  sdc: ios=61810/0, merge=0/0, ticks=864023/0, in_queue=864129, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [rrrr] [100.0% done] [264.5M/0K /s] [258 /0  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=6641
  read : io=15520MB, bw=264808KB/s, iops=258 , runt= 60015msec
    clat (usec): min=906 , max=27581 , avg=15457.23, stdev=302.14
     lat (usec): min=906 , max=27582 , avg=15457.56, stdev=302.14
    bw (KB/s) : min=65871, max=68266, per=25.04%, avg=66300.94, stdev=218.56
  cpu          : usr=0.05%, sys=0.61%, ctx=15543, majf=0, minf=1151
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=15520/0/0, short=0/0/0
     lat (usec): 1000=0.12%
     lat (msec): 10=0.03%, 20=99.85%, 50=0.01%

Run status group 0 (all jobs):
   READ: io=15520MB, aggrb=264808KB/s, minb=271163KB/s, maxb=271163KB/s, mint=60015msec, maxt=60015msec

Disk stats (read/write):
  sdc: ios=61936/0, merge=0/0, ticks=864642/0, in_queue=864747, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=1
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/89168K /s] [0 /21.8K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6663
  write: io=5058.2MB, bw=86325KB/s, iops=21581 , runt= 60001msec
    clat (usec): min=31 , max=11754 , avg=43.89, stdev=26.78
     lat (usec): min=31 , max=11755 , avg=44.18, stdev=26.84
    bw (KB/s) : min=62728, max=88088, per=100.00%, avg=86326.56, stdev=2239.73
  cpu          : usr=7.01%, sys=27.43%, ctx=1286225, majf=0, minf=29
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/1294896/0, short=0/0/0
     lat (usec): 50=94.77%, 100=0.87%, 250=4.34%, 500=0.01%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 20=0.01%

Run status group 0 (all jobs):
  WRITE: io=5058.2MB, aggrb=86324KB/s, minb=88396KB/s, maxb=88396KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/1292248, merge=0/0, ticks=0/48301, in_queue=48299, util=80.82%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/88866K /s] [0 /21.7K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6747
  write: io=5068.2MB, bw=86493KB/s, iops=21623 , runt= 60001msec
    clat (usec): min=32 , max=7763 , avg=43.79, stdev=21.34
     lat (usec): min=32 , max=7764 , avg=44.07, stdev=21.41
    bw (KB/s) : min=85192, max=88304, per=100.01%, avg=86500.77, stdev=469.53
  cpu          : usr=7.01%, sys=27.28%, ctx=1288771, majf=0, minf=27
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/1297412/0, short=0/0/0
     lat (usec): 50=94.87%, 100=0.78%, 250=4.34%, 500=0.01%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 10=0.01%

Run status group 0 (all jobs):
  WRITE: io=5068.2MB, aggrb=86492KB/s, minb=88568KB/s, maxb=88568KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/1294737, merge=0/0, ticks=0/48383, in_queue=48372, util=80.95%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/48449K /s] [0 /11.9K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6767
  write: io=2780.6MB, bw=47454KB/s, iops=11863 , runt= 60001msec
    clat (usec): min=37 , max=142538 , avg=81.04, stdev=365.04
     lat (usec): min=37 , max=142539 , avg=81.33, stdev=365.04
    bw (KB/s) : min=17952, max=54040, per=100.04%, avg=47470.49, stdev=4248.89
  cpu          : usr=4.54%, sys=13.97%, ctx=708045, majf=0, minf=2095
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/711815/0, short=0/0/0
     lat (usec): 50=42.81%, 100=40.03%, 250=15.11%, 500=1.53%, 750=0.35%
     lat (usec): 1000=0.07%
     lat (msec): 2=0.05%, 4=0.01%, 10=0.02%, 20=0.02%, 50=0.01%
     lat (msec): 100=0.01%, 250=0.01%

Run status group 0 (all jobs):
  WRITE: io=2780.6MB, aggrb=47453KB/s, minb=48592KB/s, maxb=48592KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/710262, merge=0/0, ticks=0/52729, in_queue=52726, util=88.24%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=4k --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/48710K /s] [0 /11.9K iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6783
  write: io=2891.9MB, bw=49353KB/s, iops=12338 , runt= 60001msec
    clat (usec): min=37 , max=139890 , avg=77.81, stdev=314.97
     lat (usec): min=37 , max=139890 , avg=78.11, stdev=314.98
    bw (KB/s) : min= 7208, max=54264, per=100.06%, avg=49383.99, stdev=6473.82
  cpu          : usr=4.64%, sys=14.97%, ctx=736241, majf=0, minf=1058
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/740310/0, short=0/0/0
     lat (usec): 50=40.08%, 100=43.94%, 250=14.88%, 500=0.83%, 750=0.14%
     lat (usec): 1000=0.04%
     lat (msec): 2=0.02%, 4=0.01%, 10=0.04%, 20=0.02%, 50=0.01%
     lat (msec): 250=0.01%

Run status group 0 (all jobs):
  WRITE: io=2891.9MB, aggrb=49353KB/s, minb=50537KB/s, maxb=50537KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/738727, merge=0/0, ticks=0/52339, in_queue=52334, util=87.58%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/147.2M /s] [0 /143  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6851
  write: io=5120.0MB, bw=171875KB/s, iops=167 , runt= 30504msec
    clat (usec): min=959 , max=212603 , avg=5953.52, stdev=4705.48
     lat (usec): min=960 , max=212603 , avg=5953.89, stdev=4705.49
    bw (KB/s) : min=65361, max=394942, per=99.82%, avg=171573.73, stdev=35933.48
  cpu          : usr=0.08%, sys=1.67%, ctx=5125, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/5120/0, short=0/0/0
     lat (usec): 1000=3.42%
     lat (msec): 2=2.07%, 4=0.14%, 10=93.16%, 20=1.11%, 250=0.10%

Run status group 0 (all jobs):
  WRITE: io=5120.0MB, aggrb=171875KB/s, minb=176000KB/s, maxb=176000KB/s, mint=30504msec, maxt=30504msec

Disk stats (read/write):
  sdc: ios=0/20252, merge=0/0, ticks=0/113425, in_queue=113449, util=99.49%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/187.9M /s] [0 /183  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6857
  write: io=5120.0MB, bw=175189KB/s, iops=171 , runt= 29927msec
    clat (msec): min=1 , max=630 , avg= 5.84, stdev=10.03
     lat (msec): min=1 , max=630 , avg= 5.84, stdev=10.03
    bw (KB/s) : min=70520, max=624191, per=101.53%, avg=177862.12, stdev=65043.26
  cpu          : usr=0.09%, sys=1.73%, ctx=5124, majf=0, minf=26
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/5120/0, short=0/0/0

     lat (msec): 2=8.61%, 4=0.14%, 10=91.09%, 250=0.14%, 750=0.02%

Run status group 0 (all jobs):
  WRITE: io=5120.0MB, aggrb=175188KB/s, minb=179393KB/s, maxb=179393KB/s, mint=29927msec, maxt=29927msec

Disk stats (read/write):
  sdc: ios=0/20368, merge=0/0, ticks=0/110782, in_queue=110794, util=99.45%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/168.5M /s] [0 /164  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6943
  write: io=9910.0MB, bw=169125KB/s, iops=165 , runt= 60002msec
    clat (msec): min=1 , max=650 , avg= 6.05, stdev= 8.16
     lat (msec): min=1 , max=650 , avg= 6.05, stdev= 8.16
    bw (KB/s) : min=61845, max=625437, per=100.69%, avg=170290.69, stdev=47347.73
  cpu          : usr=0.10%, sys=1.61%, ctx=9919, majf=0, minf=28
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/9910/0, short=0/0/0

     lat (msec): 2=5.38%, 4=0.17%, 10=93.72%, 20=0.60%, 250=0.11%
     lat (msec): 500=0.01%, 750=0.01%

Run status group 0 (all jobs):
  WRITE: io=9910.0MB, aggrb=169125KB/s, minb=173184KB/s, maxb=173184KB/s, mint=60002msec, maxt=60002msec

Disk stats (read/write):
  sdc: ios=0/39556, merge=0/0, ticks=0/223184, in_queue=223196, util=99.59%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/157.3M /s] [0 /153  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6961
  write: io=5120.0MB, bw=171146KB/s, iops=167 , runt= 30634msec
    clat (usec): min=770 , max=628283 , avg=5977.36, stdev=9898.81
     lat (usec): min=770 , max=628283 , avg=5977.74, stdev=9898.82
    bw (KB/s) : min=42937, max=622592, per=101.18%, avg=173164.53, stdev=64572.75
  cpu          : usr=0.10%, sys=1.61%, ctx=5123, majf=0, minf=24
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/5120/0, short=0/0/0
     lat (usec): 1000=7.73%
     lat (msec): 2=0.06%, 4=0.14%, 10=91.11%, 20=0.80%, 50=0.02%
     lat (msec): 250=0.12%, 750=0.02%

Run status group 0 (all jobs):
  WRITE: io=5120.0MB, aggrb=171145KB/s, minb=175253KB/s, maxb=175253KB/s, mint=30634msec, maxt=30634msec

Disk stats (read/write):
  sdc: ios=0/20412, merge=0/0, ticks=0/115465, in_queue=115485, util=99.46%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/168.5M /s] [0 /164  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=6975
  write: io=5120.0MB, bw=170622KB/s, iops=166 , runt= 30728msec
    clat (msec): min=1 , max=627 , avg= 6.00, stdev=10.53
     lat (msec): min=1 , max=627 , avg= 6.00, stdev=10.53
    bw (KB/s) : min=16419, max=624191, per=100.82%, avg=172013.45, stdev=66595.86
  cpu          : usr=0.13%, sys=1.62%, ctx=5128, majf=0, minf=24
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/5120/0, short=0/0/0

     lat (msec): 2=8.87%, 4=0.10%, 10=90.08%, 20=0.80%, 250=0.12%
     lat (msec): 500=0.02%, 750=0.02%

Run status group 0 (all jobs):
  WRITE: io=5120.0MB, aggrb=170622KB/s, minb=174717KB/s, maxb=174717KB/s, mint=30728msec, maxt=30728msec

Disk stats (read/write):
  sdc: ios=0/20356, merge=0/0, ticks=0/113764, in_queue=113787, util=99.44%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=159896305664 --runtime=60 --numjobs=1 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/167.5M /s] [0 /163  iops] [eta 00m:00s]
file1: (groupid=0, jobs=1): err= 0: pid=7012
  write: io=9997.0MB, bw=170604KB/s, iops=166 , runt= 60004msec
    clat (usec): min=777 , max=323097 , avg=5996.50, stdev=5234.22
     lat (usec): min=777 , max=323097 , avg=5996.88, stdev=5234.22
    bw (KB/s) : min=83159, max=434793, per=100.27%, avg=171064.50, stdev=31820.80
  cpu          : usr=0.13%, sys=1.60%, ctx=10004, majf=0, minf=38
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/9997/0, short=0/0/0
     lat (usec): 1000=4.42%
     lat (msec): 2=0.07%, 4=0.11%, 10=95.26%, 20=0.02%, 250=0.11%
     lat (msec): 500=0.01%

Run status group 0 (all jobs):
  WRITE: io=9997.0MB, aggrb=170604KB/s, minb=174698KB/s, maxb=174698KB/s, mint=60004msec, maxt=60004msec

Disk stats (read/write):
  sdc: ios=0/39904, merge=0/0, ticks=0/226430, in_queue=226434, util=99.59%
Sleeping for 30 seconds... ------------------------------------------------------
#################################################################################
Executing write tests with numjobs=4
---------------------------------------------------------------------------------
Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/200.9M /s] [0 /50.3K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7031
  write: io=11772MB, bw=200912KB/s, iops=50227 , runt= 60001msec
    clat (usec): min=33 , max=4268 , avg=76.15, stdev=14.90
     lat (usec): min=33 , max=4268 , avg=76.54, stdev=14.95
    bw (KB/s) : min=49320, max=51720, per=24.98%, avg=50194.24, stdev=175.43
  cpu          : usr=4.30%, sys=19.27%, ctx=3001355, majf=0, minf=114
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/3013724/0, short=0/0/0
     lat (usec): 50=4.40%, 100=86.11%, 250=9.46%, 500=0.03%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
  WRITE: io=11772MB, aggrb=200911KB/s, minb=205733KB/s, maxb=205733KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/3007491, merge=0/0, ticks=0/180330, in_queue=179703, util=99.04%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/200.9M /s] [0 /50.2K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7121
  write: io=11745MB, bw=200450KB/s, iops=50112 , runt= 60001msec
    clat (usec): min=33 , max=11944 , avg=76.31, stdev=18.95
     lat (usec): min=33 , max=11944 , avg=76.70, stdev=18.98
    bw (KB/s) : min=37532, max=52912, per=25.02%, avg=50148.44, stdev=639.31
  cpu          : usr=4.22%, sys=19.18%, ctx=2994515, majf=0, minf=110
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/3006802/0, short=0/0/0
     lat (usec): 50=4.42%, 100=86.01%, 250=9.53%, 500=0.03%, 750=0.01%
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%

Run status group 0 (all jobs):
  WRITE: io=11745MB, aggrb=200450KB/s, minb=205260KB/s, maxb=205260KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/3000637, merge=0/0, ticks=0/180783, in_queue=180224, util=99.10%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/49490K /s] [0 /12.9K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7137
  write: io=2779.3MB, bw=47432KB/s, iops=11857 , runt= 60001msec
    clat (usec): min=37 , max=126768 , avg=333.86, stdev=294.54
     lat (usec): min=37 , max=126769 , avg=334.17, stdev=294.53
    bw (KB/s) : min= 1025, max=16864, per=24.98%, avg=11850.18, stdev=814.22
  cpu          : usr=1.28%, sys=4.52%, ctx=716959, majf=0, minf=4233
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/711489/0, short=0/0/0
     lat (usec): 50=0.09%, 100=6.53%, 250=39.44%, 500=39.06%, 750=10.46%
     lat (usec): 1000=2.74%
     lat (msec): 2=1.48%, 4=0.06%, 10=0.03%, 20=0.10%, 50=0.01%
     lat (msec): 100=0.01%, 250=0.01%

Run status group 0 (all jobs):
  WRITE: io=2779.3MB, aggrb=47431KB/s, minb=48570KB/s, maxb=48570KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/709464, merge=0/0, ticks=0/222217, in_queue=222123, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=4k --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/50711K /s] [0 /12.4K iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7191
  write: io=2857.9MB, bw=48774KB/s, iops=12193 , runt= 60001msec
    clat (usec): min=38 , max=190031 , avg=324.25, stdev=336.95
     lat (usec): min=38 , max=190031 , avg=324.56, stdev=336.96
    bw (KB/s) : min= 1304, max=17016, per=25.01%, avg=12198.34, stdev=1058.72
  cpu          : usr=1.28%, sys=4.67%, ctx=738452, majf=0, minf=7345
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/731616/0, short=0/0/0
     lat (usec): 50=0.10%, 100=5.76%, 250=40.52%, 500=41.81%, 750=8.73%
     lat (usec): 1000=2.01%
     lat (msec): 2=0.84%, 4=0.03%, 10=0.05%, 20=0.15%, 50=0.01%
     lat (msec): 250=0.01%

Run status group 0 (all jobs):
  WRITE: io=2857.9MB, aggrb=48773KB/s, minb=49944KB/s, maxb=49944KB/s, mint=60001msec, maxt=60001msec

Disk stats (read/write):
  sdc: ios=0/729476, merge=0/0, ticks=0/222836, in_queue=222741, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 3 (f=3): [WWW_] [100.0% done] [0K/306.3M /s] [0 /299  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7214
  write: io=20480MB, bw=513442KB/s, iops=501 , runt= 40845msec
    clat (msec): min=1 , max=315 , avg= 7.83, stdev= 3.66
     lat (msec): min=1 , max=315 , avg= 7.83, stdev= 3.66
    bw (KB/s) : min=63875, max=344047, per=25.68%, avg=131866.82, stdev=23682.28
  cpu          : usr=0.05%, sys=1.04%, ctx=20493, majf=0, minf=108
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/20480/0, short=0/0/0

     lat (msec): 2=0.28%, 4=8.83%, 10=66.61%, 20=20.24%, 50=3.81%
     lat (msec): 100=0.06%, 250=0.15%, 500=0.02%

Run status group 0 (all jobs):
  WRITE: io=20480MB, aggrb=513441KB/s, minb=525764KB/s, maxb=525764KB/s, mint=40845msec, maxt=40845msec

Disk stats (read/write):
  sdc: ios=0/81845, merge=0/0, ticks=0/616836, in_queue=616890, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/653.5M /s] [0 /638  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7290
  write: io=20480MB, bw=692152KB/s, iops=675 , runt= 30299msec
    clat (usec): min=980 , max=627847 , avg=5913.97, stdev=4949.07
     lat (usec): min=981 , max=627847 , avg=5914.31, stdev=4949.07
    bw (KB/s) : min=62822, max=299260, per=25.29%, avg=175074.89, stdev=18085.36
  cpu          : usr=0.06%, sys=1.42%, ctx=20515, majf=0, minf=104
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/20480/0, short=0/0/0
     lat (usec): 1000=0.01%
     lat (msec): 2=0.01%, 4=33.14%, 10=65.81%, 20=0.89%, 250=0.12%
     lat (msec): 750=0.02%

Run status group 0 (all jobs):
  WRITE: io=20480MB, aggrb=692152KB/s, minb=708763KB/s, maxb=708763KB/s, mint=30299msec, maxt=30299msec

Disk stats (read/write):
  sdc: ios=0/81822, merge=0/0, ticks=0/443426, in_queue=443456, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/657.5M /s] [0 /642  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7309
  write: io=38944MB, bw=664545KB/s, iops=648 , runt= 60009msec
    clat (msec): min=1 , max=639 , avg= 6.16, stdev= 4.28
     lat (msec): min=1 , max=639 , avg= 6.16, stdev= 4.28
    bw (KB/s) : min=53484, max=316806, per=25.17%, avg=167293.72, stdev=14576.62
  cpu          : usr=0.08%, sys=1.32%, ctx=38973, majf=0, minf=108
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/38944/0, short=0/0/0

     lat (msec): 2=0.01%, 4=32.70%, 10=64.89%, 20=2.17%, 50=0.10%
     lat (msec): 250=0.12%, 750=0.01%

Run status group 0 (all jobs):
  WRITE: io=38944MB, aggrb=664544KB/s, minb=680493KB/s, maxb=680493KB/s, mint=60009msec, maxt=60009msec

Disk stats (read/write):
  sdc: ios=0/155426, merge=0/0, ticks=0/899224, in_queue=899290, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/187.9M /s] [0 /183  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7327
  write: io=10137MB, bw=172964KB/s, iops=168 , runt= 60014msec
    clat (msec): min=1 , max=856 , avg=23.71, stdev=11.61
     lat (msec): min=1 , max=856 , avg=23.71, stdev=11.61
    bw (KB/s) : min= 6510, max=161684, per=25.24%, avg=43648.71, stdev=6913.17
  cpu          : usr=0.02%, sys=0.39%, ctx=10151, majf=0, minf=112
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/10137/0, short=0/0/0

     lat (msec): 2=0.02%, 4=7.17%, 10=0.36%, 20=17.96%, 50=73.39%
     lat (msec): 100=0.10%, 250=0.94%, 500=0.01%, 1000=0.04%

Run status group 0 (all jobs):
  WRITE: io=10137MB, aggrb=172964KB/s, minb=177115KB/s, maxb=177115KB/s, mint=60014msec, maxt=60014msec

Disk stats (read/write):
  sdc: ios=0/40452, merge=0/0, ticks=0/914786, in_queue=914830, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=154527596544 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/152.2M /s] [0 /148  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7383
  write: io=10143MB, bw=173058KB/s, iops=169 , runt= 60017msec
    clat (msec): min=1 , max=652 , avg=23.75, stdev= 8.60
     lat (msec): min=1 , max=652 , avg=23.75, stdev= 8.60
    bw (KB/s) : min=15481, max=161056, per=25.03%, avg=43309.68, stdev=5974.14
  cpu          : usr=0.02%, sys=0.40%, ctx=10163, majf=0, minf=104
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/10143/0, short=0/0/0

     lat (msec): 2=0.01%, 4=4.55%, 10=0.32%, 20=27.34%, 50=67.06%
     lat (msec): 100=0.14%, 250=0.54%, 750=0.04%

Run status group 0 (all jobs):
  WRITE: io=10143MB, aggrb=173058KB/s, minb=177211KB/s, maxb=177211KB/s, mint=60017msec, maxt=60017msec

Disk stats (read/write):
  sdc: ios=0/40400, merge=0/0, ticks=0/890035, in_queue=890267, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------
---------------------------------------------------------------------------------
Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=159896305664 --runtime=60 --numjobs=4 --group_reporting --name=file1
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
...
file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1
fio 1.54
Starting 4 processes
Jobs: 4 (f=4): [wwww] [100.0% done] [0K/167.5M /s] [0 /163  iops] [eta 00m:00s]
file1: (groupid=0, jobs=4): err= 0: pid=7402
  write: io=9841.0MB, bw=167886KB/s, iops=163 , runt= 60024msec
    clat (msec): min=1 , max=1235 , avg=24.38, stdev=13.42
     lat (msec): min=1 , max=1235 , avg=24.38, stdev=13.42
    bw (KB/s) : min= 9709, max=158000, per=25.42%, avg=42674.97, stdev=6190.60
  cpu          : usr=0.02%, sys=0.36%, ctx=9850, majf=0, minf=150
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued r/w/d: total=0/9841/0, short=0/0/0

     lat (msec): 2=0.01%, 4=5.71%, 10=0.16%, 20=0.85%, 50=92.61%
     lat (msec): 250=0.61%, 2000=0.04%

Run status group 0 (all jobs):
  WRITE: io=9841.0MB, aggrb=167885KB/s, minb=171915KB/s, maxb=171915KB/s, mint=60024msec, maxt=60024msec

Disk stats (read/write):
  sdc: ios=0/39264, merge=0/0, ticks=0/942957, in_queue=943282, util=100.00%
Sleeping for 30 seconds... ------------------------------------------------------