[root@iotest ~]# cat fiotester2.sh #!/bin/bash FIO="/usr/bin/fio" FN="/dev/sdc" OFFSET="174598389760" # 5GB are 5.368.709.120 Bytes DEVSIZE="179967098880" DEVSIZE10P="17996709888" BSIOPS="512" BSTHR="1m" for TESTCASE in "read" "write" do for NJ in 1 4 64 do echo "#################################################################################" echo "Executing $TESTCASE tests with numjobs=$NJ" for testrun in `seq 1 10` do case $testrun in 1) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 2) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 3) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 4) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 5) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 6) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";; 7) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 8) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 9) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";; 10) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; esac echo "---------------------------------------------------------------------------------" echo "Executing test $testrun: $TESTCMD" $TESTCMD echo "Sleeping for 30 seconds... ------------------------------------------------------" sleep 30 done done done [root@iotest ~]# ./fiotester2.sh ################################################################################# Executing read tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=512 --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [16M/0K /s] [32K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25629 read : io=976MB, bw=16,655KB/s, iops=33,310, runt= 60001msec clat (usec): min=18, max=2,627, avg=28.10, stdev= 7.44 bw (KB/s) : min=15198, max=16997, per=100.01%, avg=16656.94, stdev=237.14 cpu : usr=9.38%, sys=21.80%, ctx=1999253, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1998667/0, short=0/0 lat (usec): 20=0.08%, 50=99.05%, 100=0.82%, 250=0.03%, 500=0.02% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): READ: io=976MB, aggrb=16,655KB/s, minb=17,055KB/s, maxb=17,055KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=1995775/0, merge=0/0, ticks=25645/0, in_queue=25677, util=42.90% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=512 --size=17996709888 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [17M/0K /s] [33K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25634 read : io=974MB, bw=16,618KB/s, iops=33,235, runt= 60001msec clat (usec): min=18, max=2,632, avg=28.18, stdev= 6.80 bw (KB/s) : min=15036, max=16890, per=100.01%, avg=16617.94, stdev=204.20 cpu : usr=9.67%, sys=23.18%, ctx=1994810, majf=0, minf=27 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1994180/0, short=0/0 lat (usec): 20=0.07%, 50=99.05%, 100=0.82%, 250=0.03%, 500=0.02% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): READ: io=974MB, aggrb=16,617KB/s, minb=17,016KB/s, maxb=17,016KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=1991199/0, merge=0/0, ticks=24609/0, in_queue=24444, util=40.85% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=512 --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [4,160K/0K /s] [8K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25639 read : io=240MB, bw=4,098KB/s, iops=8,196, runt= 60001msec clat (usec): min=50, max=5,120, avg=118.99, stdev=13.39 bw (KB/s) : min= 3960, max= 4232, per=100.03%, avg=4099.31, stdev=47.99 cpu : usr=2.66%, sys=6.14%, ctx=492011, majf=0, minf=3030 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=491791/0, short=0/0 lat (usec): 100=0.25%, 250=99.73%, 500=0.02%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): READ: io=240MB, aggrb=4,098KB/s, minb=4,196KB/s, maxb=4,196KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=490672/0, merge=0/0, ticks=45546/0, in_queue=45479, util=76.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=512 --size=17996709888 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [4,308K/0K /s] [8K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25646 read : io=245MB, bw=4,178KB/s, iops=8,356, runt= 60001msec clat (usec): min=32, max=2,016, avg=116.72, stdev=15.16 bw (KB/s) : min= 4034, max= 4292, per=100.02%, avg=4178.93, stdev=40.28 cpu : usr=2.75%, sys=6.43%, ctx=501623, majf=0, minf=1042 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=501388/0, short=0/0 lat (usec): 50=0.01%, 100=1.10%, 250=98.86%, 500=0.01%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): READ: io=245MB, aggrb=4,178KB/s, minb=4,278KB/s, maxb=4,278KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=500286/0, merge=0/0, ticks=47614/0, in_queue=47556, util=79.47% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [466M/0K /s] [455/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25681 read : io=5,120MB, bw=463MB/s, iops=463, runt= 11051msec clat (usec): min=473, max=3,616, avg=2155.13, stdev=386.65 bw (KB/s) : min=458752, max=717413, per=100.09%, avg=474835.32, stdev=54247.30 cpu : usr=0.15%, sys=5.15%, ctx=5146, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (usec): 500=2.95%, 750=0.06%, 1000=0.02% lat (msec): 2=0.02%, 4=96.95% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=463MB/s, minb=474MB/s, maxb=474MB/s, mint=11051msec, maxt=11051msec Disk stats (read/write): sdc: ios=40590/0, merge=0/0, ticks=47505/0, in_queue=47507, util=97.35% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [425M/0K /s] [414/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25689 read : io=5,120MB, bw=449MB/s, iops=449, runt= 11396msec clat (usec): min=522, max=5,724, avg=2222.68, stdev=490.91 bw (KB/s) : min=414914, max=665600, per=100.32%, avg=461536.36, stdev=60278.96 cpu : usr=0.21%, sys=5.12%, ctx=5159, majf=0, minf=280 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (usec): 750=0.74%, 1000=2.70% lat (msec): 2=5.12%, 4=91.15%, 10=0.29% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=449MB/s, minb=460MB/s, maxb=460MB/s, mint=11396msec, maxt=11396msec Disk stats (read/write): sdc: ios=40303/0, merge=0/0, ticks=48937/0, in_queue=48937, util=97.04% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [375M/0K /s] [365/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25694 read : io=23,993MB, bw=400MB/s, iops=399, runt= 60002msec clat (usec): min=496, max=4,910, avg=2497.70, stdev=582.12 bw (KB/s) : min=320894, max=856064, per=100.16%, avg=410122.22, stdev=66078.89 cpu : usr=0.16%, sys=4.62%, ctx=24140, majf=0, minf=284 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=23993/0, short=0/0 lat (usec): 500=0.01%, 750=0.14%, 1000=0.63% lat (msec): 2=1.55%, 4=96.37%, 10=1.31% Run status group 0 (all jobs): READ: io=23,993MB, aggrb=400MB/s, minb=409MB/s, maxb=409MB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdc: ios=191697/0, merge=0/0, ticks=256263/0, in_queue=256262, util=98.11% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [305M/0K /s] [298/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25699 read : io=5,120MB, bw=273MB/s, iops=272, runt= 18756msec clat (usec): min=486, max=7,280, avg=3658.73, stdev=1070.14 bw (KB/s) : min=265708, max=318215, per=99.98%, avg=279485.54, stdev=12460.75 cpu : usr=0.15%, sys=3.31%, ctx=5152, majf=0, minf=277 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (usec): 500=0.08%, 750=2.97%, 1000=0.16% lat (msec): 2=0.98%, 4=59.71%, 10=36.11% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=273MB/s, minb=280MB/s, maxb=280MB/s, mint=18756msec, maxt=18756msec Disk stats (read/write): sdc: ios=40908/0, merge=0/0, ticks=106453/0, in_queue=106468, util=98.41% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [281M/0K /s] [274/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25704 read : io=5,120MB, bw=246MB/s, iops=246, runt= 20774msec clat (usec): min=481, max=9,141, avg=4052.82, stdev=1195.81 bw (KB/s) : min=233005, max=292279, per=100.02%, avg=252431.93, stdev=12338.56 cpu : usr=0.14%, sys=3.04%, ctx=5148, majf=0, minf=277 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (usec): 500=0.16%, 750=2.79%, 1000=0.18% lat (msec): 2=0.82%, 4=48.32%, 10=47.73% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=246MB/s, minb=252MB/s, maxb=252MB/s, mint=20774msec, maxt=20774msec Disk stats (read/write): sdc: ios=40882/0, merge=0/0, ticks=115880/0, in_queue=115887, util=98.64% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [179M/0K /s] [174/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=25707 read : io=10,217MB, bw=170MB/s, iops=170, runt= 60006msec clat (usec): min=519, max=14,660, avg=5868.31, stdev=2176.71 bw (KB/s) : min=161469, max=195047, per=100.09%, avg=174515.89, stdev=5830.05 cpu : usr=0.10%, sys=2.11%, ctx=10279, majf=0, minf=298 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=10217/0, short=0/0 lat (usec): 750=0.06% lat (msec): 4=20.10%, 10=74.30%, 20=5.54% Run status group 0 (all jobs): READ: io=10,217MB, aggrb=170MB/s, minb=174MB/s, maxb=174MB/s, mint=60006msec, maxt=60006msec Disk stats (read/write): sdc: ios=81608/0, merge=0/0, ticks=317754/0, in_queue=317754, util=99.38% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=512 --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [17M/0K /s] [34K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25715 read : io=1,743MB, bw=29,743KB/s, iops=59,486, runt= 60001msec clat (usec): min=18, max=2,682, avg=64.73, stdev=22.23 bw (KB/s) : min= 3203, max=13700, per=24.84%, avg=7389.18, stdev=2145.27 cpu : usr=4.22%, sys=19.06%, ctx=3566347, majf=0, minf=118 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=3569274/0, short=0/0 lat (usec): 20=0.01%, 50=54.43%, 100=11.28%, 250=34.24%, 500=0.02% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): READ: io=1,743MB, aggrb=29,743KB/s, minb=30,457KB/s, maxb=30,457KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=3559247/0, merge=532/0, ticks=173883/0, in_queue=173654, util=98.46% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=512 --size=17996709888 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [51M/0K /s] [102K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25723 read : io=3,035MB, bw=51,794KB/s, iops=104K, runt= 60001msec clat (usec): min=17, max=101K, avg=36.46, stdev=35.99 bw (KB/s) : min=10544, max=13699, per=24.96%, avg=12925.88, stdev=161.37 cpu : usr=5.46%, sys=30.51%, ctx=6203115, majf=0, minf=112 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=6215328/0, short=0/0 lat (usec): 20=0.01%, 50=83.51%, 100=16.10%, 250=0.35%, 500=0.03% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=3,035MB, aggrb=51,793KB/s, minb=53,036KB/s, maxb=53,036KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=6205008/0, merge=83/0, ticks=124299/0, in_queue=124386, util=94.88% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=512 --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [15M/0K /s] [29K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25731 read : io=850MB, bw=14,501KB/s, iops=29,002, runt= 60001msec clat (usec): min=55, max=164K, avg=133.47, stdev=125.49 bw (KB/s) : min= 2158, max= 3738, per=24.98%, avg=3622.14, stdev=74.64 cpu : usr=3.53%, sys=10.27%, ctx=1751490, majf=0, minf=15727 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1740157/0, short=0/0 lat (usec): 100=4.19%, 250=95.70%, 500=0.06%, 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.01%, 10=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=850MB, aggrb=14,501KB/s, minb=14,849KB/s, maxb=14,849KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=1734740/0, merge=0/0, ticks=196449/0, in_queue=196584, util=99.73% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=512 --size=17996709888 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [14M/0K /s] [29K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25768 read : io=844MB, bw=14,398KB/s, iops=28,796, runt= 60001msec clat (usec): min=45, max=12,152, avg=134.38, stdev=19.41 bw (KB/s) : min= 3323, max= 3662, per=24.99%, avg=3598.36, stdev=20.46 cpu : usr=3.48%, sys=10.30%, ctx=1739253, majf=0, minf=11877 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1727814/0, short=0/0 lat (usec): 50=0.01%, 100=4.21%, 250=95.63%, 500=0.10%, 750=0.01% lat (usec): 1000=0.02% lat (msec): 2=0.03%, 4=0.01%, 10=0.01%, 20=0.01% Run status group 0 (all jobs): READ: io=844MB, aggrb=14,398KB/s, minb=14,743KB/s, maxb=14,743KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=1722444/0, merge=0/0, ticks=197140/0, in_queue=197291, util=99.83% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [2,248M/0K /s] [2K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25776 read : io=20,480MB, bw=2,103MB/s, iops=2,103, runt= 9737msec clat (usec): min=519, max=426K, avg=1898.16, stdev=2023.88 bw (KB/s) : min=197471, max=562075, per=25.00%, avg=538383.28, stdev=41058.41 cpu : usr=0.24%, sys=7.42%, ctx=20627, majf=0, minf=1116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (usec): 750=0.50%, 1000=0.15% lat (msec): 2=95.85%, 4=2.99%, 10=0.46%, 100=0.03%, 250=0.01% lat (msec): 500=0.01% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=2,103MB/s, minb=2,154MB/s, maxb=2,154MB/s, mint=9737msec, maxt=9737msec Disk stats (read/write): sdc: ios=162982/0, merge=0/0, ticks=259283/0, in_queue=259329, util=99.21% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 3 (f=3): [RRR_] [95.7% done] [992M/0K /s] [969/0 iops] [eta 00m:01s] file1: (groupid=0, jobs=4): err= 0: pid=25784 read : io=20,480MB, bw=938MB/s, iops=938, runt= 21827msec clat (usec): min=471, max=303K, avg=2647.02, stdev=3442.28 bw (KB/s) : min= 3785, max=726183, per=44.57%, avg=428211.85, stdev=78590.40 cpu : usr=0.10%, sys=3.46%, ctx=20643, majf=0, minf=1116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (usec): 500=0.16%, 750=6.51%, 1000=15.59% lat (msec): 2=35.96%, 4=17.87%, 10=20.05%, 20=2.99%, 50=0.01% lat (msec): 100=0.02%, 250=0.83%, 500=0.01% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=938MB/s, minb=961MB/s, maxb=961MB/s, mint=21827msec, maxt=21827msec Disk stats (read/write): sdc: ios=163204/0, merge=0/0, ticks=428193/0, in_queue=428234, util=99.81% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [1,064M/0K /s] [1K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25792 read : io=77,640MB, bw=1,294MB/s, iops=1,293, runt= 60003msec clat (usec): min=493, max=257K, avg=3098.12, stdev=1402.28 bw (KB/s) : min=227328, max=558916, per=24.96%, avg=330760.00, stdev=41308.84 cpu : usr=0.15%, sys=5.06%, ctx=78477, majf=0, minf=1128 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=77640/0, short=0/0 lat (usec): 500=0.01%, 750=2.02%, 1000=6.94% lat (msec): 2=33.12%, 4=31.61%, 10=25.28%, 20=1.03%, 50=0.01% lat (msec): 250=0.01%, 500=0.01% Run status group 0 (all jobs): READ: io=77,640MB, aggrb=1,294MB/s, minb=1,325MB/s, maxb=1,325MB/s, mint=60003msec, maxt=60003msec Disk stats (read/write): sdc: ios=620340/0, merge=0/0, ticks=1163545/0, in_queue=1163609, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [361M/0K /s] [352/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25800 read : io=18,338MB, bw=306MB/s, iops=305, runt= 60011msec clat (usec): min=516, max=38,118, avg=13123.53, stdev=1927.45 bw (KB/s) : min=68400, max=97912, per=24.96%, avg=78096.24, stdev=2544.70 cpu : usr=0.04%, sys=0.96%, ctx=18463, majf=0, minf=1148 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=18338/0, short=0/0 lat (usec): 750=1.46%, 1000=0.71% lat (msec): 2=0.32%, 4=1.04%, 10=9.52%, 20=83.16%, 50=3.80% Run status group 0 (all jobs): READ: io=18,338MB, aggrb=306MB/s, minb=313MB/s, maxb=313MB/s, mint=60011msec, maxt=60011msec Disk stats (read/write): sdc: ios=146421/0, merge=0/0, ticks=1279740/0, in_queue=1279911, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [292M/0K /s] [284/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25808 read : io=15,962MB, bw=266MB/s, iops=265, runt= 60017msec clat (usec): min=520, max=41,049, avg=15061.47, stdev=2061.20 bw (KB/s) : min=59076, max=85164, per=24.98%, avg=68042.12, stdev=1833.55 cpu : usr=0.04%, sys=0.81%, ctx=16067, majf=0, minf=1132 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=15962/0, short=0/0 lat (usec): 750=1.31%, 1000=0.53% lat (msec): 2=0.31%, 4=0.78%, 10=2.72%, 20=85.46%, 50=8.90% Run status group 0 (all jobs): READ: io=15,962MB, aggrb=266MB/s, minb=272MB/s, maxb=272MB/s, mint=60017msec, maxt=60017msec Disk stats (read/write): sdc: ios=127466/0, merge=0/0, ticks=1305413/0, in_queue=1305612, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [184M/0K /s] [179/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=25816 read : io=10,543MB, bw=176MB/s, iops=175, runt= 60025msec clat (usec): min=530, max=57,275, avg=22765.22, stdev=3050.44 bw (KB/s) : min=36717, max=51596, per=25.03%, avg=45016.04, stdev=1281.21 cpu : usr=0.03%, sys=0.54%, ctx=10623, majf=0, minf=1248 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=10543/0, short=0/0 lat (usec): 750=0.04%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.09%, 20=35.82%, 50=63.95% lat (msec): 100=0.09% Run status group 0 (all jobs): READ: io=10,543MB, aggrb=176MB/s, minb=180MB/s, maxb=180MB/s, mint=60025msec, maxt=60025msec Disk stats (read/write): sdc: ios=84198/0, merge=0/0, ticks=1125797/0, in_queue=1125905, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=512 --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [67M/0K /s] [134K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=25825 read : io=3,802MB, bw=64,884KB/s, iops=130K, runt= 60002msec clat (usec): min=21, max=414K, avg=491.49, stdev=136.36 bw (KB/s) : min= 134, max= 1066, per=1.56%, avg=1013.42, stdev=11.41 cpu : usr=0.38%, sys=2.35%, ctx=7791235, majf=0, minf=1845 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=7786395/0, short=0/0 lat (usec): 50=0.01%, 100=0.01%, 250=0.01%, 500=76.54%, 750=22.82% lat (usec): 1000=0.41% lat (msec): 2=0.19%, 4=0.02%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01%, 500=0.01% Run status group 0 (all jobs): READ: io=3,802MB, aggrb=64,884KB/s, minb=66,441KB/s, maxb=66,441KB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdc: ios=7768108/0, merge=4428/0, ticks=3733980/0, in_queue=3732723, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=512 --size=17996709888 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [67M/0K /s] [133K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=25895 read : io=3,834MB, bw=65,424KB/s, iops=131K, runt= 60002msec clat (usec): min=20, max=773K, avg=489.44, stdev=190.75 bw (KB/s) : min= 0, max= 6323, per=1.56%, avg=1022.16, stdev=12.07 cpu : usr=0.37%, sys=2.33%, ctx=7855533, majf=0, minf=2037 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=7851085/0, short=0/0 lat (usec): 50=0.52%, 100=0.06%, 250=0.02%, 500=80.72%, 750=18.09% lat (usec): 1000=0.32% lat (msec): 2=0.22%, 4=0.02%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01% Run status group 0 (all jobs): READ: io=3,834MB, aggrb=65,423KB/s, minb=66,993KB/s, maxb=66,993KB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdc: ios=7833623/0, merge=3150/0, ticks=3740811/0, in_queue=3740095, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=512 --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 4 (f=4): [_________rrrEr__________________________________________________] [1.7% done] [22M/0K /s] [43K/0 iops] [eta 59m:00s]s] file1: (groupid=0, jobs=64): err= 0: pid=25992 read : io=1,307MB, bw=22,299KB/s, iops=44,598, runt= 60002msec clat (usec): min=65, max=164K, avg=1457.44, stdev=112.07 bw (KB/s) : min= 183, max= 2852, per=1.53%, avg=340.76, stdev= 4.97 cpu : usr=0.35%, sys=1.04%, ctx=2701229, majf=0, minf=48196 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=2675995/0, short=0/0 lat (usec): 100=0.02%, 250=0.96%, 500=1.32%, 750=1.43%, 1000=1.41% lat (msec): 2=93.82%, 4=0.48%, 10=0.53%, 20=0.03%, 50=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=1,307MB, aggrb=22,299KB/s, minb=22,834KB/s, maxb=22,834KB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdc: ios=2673611/0, merge=0/0, ticks=3757626/0, in_queue=3757632, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=512 --size=17996709888 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 4 (f=4): [_______________________________________Errrr____________________] [1.7% done] [19M/0K /s] [39K/0 iops] [eta 59m:00s]s] file1: (groupid=0, jobs=64): err= 0: pid=26060 read : io=1,247MB, bw=21,278KB/s, iops=42,555, runt= 60008msec clat (usec): min=56, max=169K, avg=1510.84, stdev=123.63 bw (KB/s) : min= 192, max= 2882, per=1.53%, avg=324.98, stdev= 4.63 cpu : usr=0.33%, sys=1.03%, ctx=2617538, majf=0, minf=148986 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=2553645/0, short=0/0 lat (usec): 100=0.03%, 250=0.90%, 500=1.28%, 750=1.26%, 1000=1.29% lat (msec): 2=94.08%, 4=0.65%, 10=0.40%, 20=0.11%, 250=0.01% Run status group 0 (all jobs): READ: io=1,247MB, aggrb=21,277KB/s, minb=21,788KB/s, maxb=21,788KB/s, mint=60008msec, maxt=60008msec Disk stats (read/write): sdc: ios=2550679/0, merge=0/0, ticks=3758907/0, in_queue=3758918, util=99.99% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [2,474M/0K /s] [2K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26128 read : io=144GB, bw=2,454MB/s, iops=2,453, runt= 60030msec clat (usec): min=496, max=441K, avg=26877.41, stdev=3703.17 bw (KB/s) : min= 3637, max=69079, per=1.52%, avg=38130.39, stdev=610.16 cpu : usr=0.02%, sys=0.66%, ctx=194524, majf=0, minf=18165 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147304/0, short=0/0 lat (usec): 500=0.01%, 750=0.02%, 1000=0.01% lat (msec): 2=0.01%, 4=0.03%, 10=69.24%, 20=0.57%, 50=0.35% lat (msec): 100=29.31%, 250=0.42%, 500=0.06% Run status group 0 (all jobs): READ: io=144GB, aggrb=2,454MB/s, minb=2,513MB/s, maxb=2,513MB/s, mint=60030msec, maxt=60030msec Disk stats (read/write): sdc: ios=1176005/0, merge=0/0, ticks=8181936/0, in_queue=8182379, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [2,509M/0K /s] [2K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26198 read : io=144GB, bw=2,454MB/s, iops=2,453, runt= 60025msec clat (usec): min=514, max=485K, avg=25545.75, stdev=3751.03 bw (KB/s) : min= 3324, max=69632, per=1.60%, avg=40129.11, stdev=747.25 cpu : usr=0.02%, sys=0.66%, ctx=194687, majf=0, minf=18357 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=147290/0, short=0/0 lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=69.11%, 20=0.61%, 50=0.30% lat (msec): 100=29.48%, 250=0.42%, 500=0.05% Run status group 0 (all jobs): READ: io=144GB, aggrb=2,454MB/s, minb=2,513MB/s, maxb=2,513MB/s, mint=60025msec, maxt=60025msec Disk stats (read/write): sdc: ios=1175989/0, merge=0/0, ticks=8184024/0, in_queue=8184417, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=read --bs=1m --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [2,551M/0K /s] [2K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26269 read : io=145GB, bw=2,478MB/s, iops=2,478, runt= 60028msec clat (usec): min=519, max=354K, avg=25665.19, stdev=3606.69 bw (KB/s) : min= 6131, max=69354, per=1.57%, avg=39912.22, stdev=591.64 cpu : usr=0.02%, sys=0.66%, ctx=195624, majf=0, minf=18165 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=148757/0, short=0/0 lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=69.43%, 20=0.53%, 50=0.22% lat (msec): 100=29.51%, 250=0.24%, 500=0.03% Run status group 0 (all jobs): READ: io=145GB, aggrb=2,478MB/s, minb=2,538MB/s, maxb=2,538MB/s, mint=60028msec, maxt=60028msec Disk stats (read/write): sdc: ios=1187422/0, merge=0/0, ticks=8165035/0, in_queue=8165424, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr] [100.0% done] [493M/0K /s] [481/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26337 read : io=28,700MB, bw=477MB/s, iops=477, runt= 60121msec clat (usec): min=575, max=703K, avg=137380.71, stdev=6472.52 bw (KB/s) : min= 1556, max=15544, per=1.53%, avg=7481.09, stdev=188.55 cpu : usr=0.00%, sys=0.10%, ctx=52911, majf=0, minf=18695 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=28700/0, short=0/0 lat (usec): 750=0.05%, 1000=0.04% lat (msec): 2=0.09%, 4=0.07%, 10=0.14%, 20=1.14%, 50=3.78% lat (msec): 100=19.91%, 250=74.43%, 500=0.23%, 750=0.13% Run status group 0 (all jobs): READ: io=28,700MB, aggrb=477MB/s, minb=489MB/s, maxb=489MB/s, mint=60121msec, maxt=60121msec Disk stats (read/write): sdc: ios=228949/0, merge=0/0, ticks=7985751/0, in_queue=7990745, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 2 (f=2): [__________E________r_r_E___________________________E___________E] [7.7% done] [444M/0K /s] [433/0 iops] [eta 12m:09s] s] file1: (groupid=0, jobs=64): err= 0: pid=26405 read : io=25,817MB, bw=429MB/s, iops=429, runt= 60151msec clat (usec): min=654, max=781K, avg=154086.80, stdev=7099.77 bw (KB/s) : min= 1412, max=15574, per=1.52%, avg=6689.27, stdev=181.87 cpu : usr=0.00%, sys=0.09%, ctx=48072, majf=0, minf=18039 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=25817/0, short=0/0 lat (usec): 750=0.01%, 1000=0.02% lat (msec): 2=0.05%, 4=0.05%, 10=0.11%, 20=0.82%, 50=2.91% lat (msec): 100=14.77%, 250=79.90%, 500=1.17%, 750=0.19%, 1000=0.01% Run status group 0 (all jobs): READ: io=25,817MB, aggrb=429MB/s, minb=440MB/s, maxb=440MB/s, mint=60151msec, maxt=60151msec Disk stats (read/write): sdc: ios=206530/0, merge=0/0, ticks=8007689/0, in_queue=8008429, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randread --bs=1m --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 13 (f=13): [_rr__r___r_rr_r__________r_______r_____r_________r_r_________r__] [2.0% done] [328M/0K /s] [320/0 iops] [eta 50m:59s] file1: (groupid=0, jobs=64): err= 0: pid=26473 read : io=18,424MB, bw=306MB/s, iops=306, runt= 60191msec clat (usec): min=935, max=832K, avg=216587.15, stdev=9679.93 bw (KB/s) : min= 1306, max=12239, per=1.51%, avg=4739.91, stdev=130.46 cpu : usr=0.00%, sys=0.07%, ctx=34799, majf=0, minf=20347 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=18424/0, short=0/0 lat (usec): 1000=0.01% lat (msec): 10=0.01%, 20=0.31%, 50=2.02%, 100=3.50%, 250=63.30% lat (msec): 500=30.49%, 750=0.29%, 1000=0.06% Run status group 0 (all jobs): READ: io=18,424MB, aggrb=306MB/s, minb=313MB/s, maxb=313MB/s, mint=60191msec, maxt=60191msec Disk stats (read/write): sdc: ios=147338/0, merge=0/0, ticks=7980215/0, in_queue=7986668, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=512 --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/16M /s] [0/32K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26544 write: io=927MB, bw=15,814KB/s, iops=31,628, runt= 60000msec clat (usec): min=19, max=27,583, avg=29.62, stdev=23.80 bw (KB/s) : min=14623, max=16136, per=100.01%, avg=15814.82, stdev=235.28 cpu : usr=10.91%, sys=22.69%, ctx=1898430, majf=0, minf=23 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1897713, short=0/0 lat (usec): 20=0.01%, 50=99.01%, 100=0.86%, 250=0.13%, 500=0.01% lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 10=0.01%, 50=0.01% Run status group 0 (all jobs): WRITE: io=927MB, aggrb=15,814KB/s, minb=16,193KB/s, maxb=16,193KB/s, mint=60000msec, maxt=60000msec Disk stats (read/write): sdc: ios=0/1894904, merge=0/0, ticks=0/22233, in_queue=22687, util=37.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=512 --size=17996709888 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/16M /s] [0/32K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26578 write: io=925MB, bw=15,779KB/s, iops=31,557, runt= 60001msec clat (usec): min=20, max=1,692, avg=29.68, stdev= 7.82 bw (KB/s) : min=14299, max=16097, per=100.01%, avg=15778.82, stdev=251.76 cpu : usr=10.94%, sys=22.02%, ctx=1894134, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1893471, short=0/0 lat (usec): 50=98.98%, 100=0.89%, 250=0.13%, 500=0.01%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01% Run status group 0 (all jobs): WRITE: io=925MB, aggrb=15,778KB/s, minb=16,157KB/s, maxb=16,157KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=0/1890463, merge=0/0, ticks=0/22602, in_queue=22946, util=38.34% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=512 --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/1,478K /s] [0/3K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26583 write: io=82,902KB, bw=1,382KB/s, iops=2,763, runt= 60001msec clat (usec): min=28, max=9,149, avg=358.14, stdev=318.06 bw (KB/s) : min= 232, max= 1450, per=100.07%, avg=1382.02, stdev=222.15 cpu : usr=1.18%, sys=2.73%, ctx=165969, majf=0, minf=1039 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/165803, short=0/0 lat (usec): 50=0.60%, 100=0.24%, 250=3.41%, 500=93.14%, 750=2.12% lat (usec): 1000=0.18% lat (msec): 2=0.07%, 4=0.01%, 10=0.25% Run status group 0 (all jobs): WRITE: io=82,901KB, aggrb=1,381KB/s, minb=1,414KB/s, maxb=1,414KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=0/165401, merge=0/0, ticks=0/56577, in_queue=56603, util=94.59% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=512 --size=17996709888 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/1,187K /s] [0/2K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26588 write: io=83,627KB, bw=1,393KB/s, iops=2,785, runt= 60035msec clat (usec): min=28, max=263K, avg=355.32, stdev=2273.53 bw (KB/s) : min= 712, max= 2263, per=100.06%, avg=1392.81, stdev=146.47 cpu : usr=1.07%, sys=2.60%, ctx=167424, majf=0, minf=3115 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/167254, short=0/0 lat (usec): 50=15.88%, 100=1.89%, 250=3.62%, 500=72.98%, 750=3.96% lat (usec): 1000=1.13% lat (msec): 2=0.45%, 4=0.02%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.03%, 250=0.02%, 500=0.01% Run status group 0 (all jobs): WRITE: io=83,627KB, aggrb=1,392KB/s, minb=1,426KB/s, maxb=1,426KB/s, mint=60035msec, maxt=60035msec Disk stats (read/write): sdc: ios=0/166689, merge=0/0, ticks=0/56551, in_queue=56644, util=94.66% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/303M /s] [0/296 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26593 write: io=5,120MB, bw=478MB/s, iops=478, runt= 10711msec clat (usec): min=442, max=73,428, avg=2089.36, stdev=5265.25 bw (KB/s) : min=289518, max=901365, per=100.66%, avg=492722.95, stdev=205433.80 cpu : usr=0.24%, sys=4.77%, ctx=5148, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (usec): 500=1.52%, 750=0.06%, 1000=8.24% lat (msec): 2=85.70%, 4=1.02%, 10=0.78%, 20=0.37%, 50=2.07% lat (msec): 100=0.23% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=478MB/s, minb=489MB/s, maxb=489MB/s, mint=10711msec, maxt=10711msec Disk stats (read/write): sdc: ios=0/40881, merge=0/0, ticks=0/49576, in_queue=49822, util=98.25% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/691M /s] [0/674 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26598 write: io=5,120MB, bw=676MB/s, iops=676, runt= 7572msec clat (usec): min=423, max=48,429, avg=1476.37, stdev=1363.95 bw (KB/s) : min=588646, max=899072, per=100.21%, avg=693832.87, stdev=71293.49 cpu : usr=0.22%, sys=6.66%, ctx=5145, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (usec): 500=1.52%, 750=0.08%, 1000=2.58% lat (msec): 2=90.80%, 4=1.02%, 10=3.57%, 20=0.39%, 50=0.04% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=676MB/s, minb=692MB/s, maxb=692MB/s, mint=7572msec, maxt=7572msec Disk stats (read/write): sdc: ios=0/40272, merge=0/0, ticks=0/33824, in_queue=33824, util=96.85% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/159M /s] [0/155 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26601 write: io=18,822MB, bw=313MB/s, iops=313, runt= 60043msec clat (usec): min=427, max=151K, avg=3187.42, stdev=11283.78 bw (KB/s) : min=111524, max=905216, per=101.74%, avg=326580.10, stdev=195936.03 cpu : usr=0.15%, sys=3.18%, ctx=18926, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/18822, short=0/0 lat (usec): 500=0.41%, 750=0.15%, 1000=9.82% lat (msec): 2=83.72%, 4=0.55%, 10=1.51%, 20=0.92%, 50=1.01% lat (msec): 100=1.68%, 250=0.22% Run status group 0 (all jobs): WRITE: io=18,822MB, aggrb=313MB/s, minb=321MB/s, maxb=321MB/s, mint=60043msec, maxt=60043msec Disk stats (read/write): sdc: ios=0/150455, merge=0/0, ticks=0/289838, in_queue=289820, util=99.29% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/286M /s] [0/279 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26608 write: io=5,120MB, bw=342MB/s, iops=342, runt= 14952msec clat (usec): min=426, max=123K, avg=2915.78, stdev=6098.67 bw (KB/s) : min=210522, max=617587, per=101.12%, avg=354575.90, stdev=66877.11 cpu : usr=0.16%, sys=3.30%, ctx=5142, majf=0, minf=20 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (usec): 500=1.52%, 750=0.02%, 1000=0.76% lat (msec): 2=66.35%, 4=25.64%, 10=0.70%, 20=2.44%, 50=2.29% lat (msec): 100=0.20%, 250=0.08% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=342MB/s, minb=351MB/s, maxb=351MB/s, mint=14952msec, maxt=14952msec Disk stats (read/write): sdc: ios=0/40753, merge=0/0, ticks=0/65773, in_queue=65943, util=98.73% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/263M /s] [0/257 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26611 write: io=5,120MB, bw=352MB/s, iops=351, runt= 14552msec clat (usec): min=414, max=124K, avg=2837.81, stdev=5863.89 bw (KB/s) : min=224054, max=655360, per=101.00%, avg=363875.86, stdev=70378.03 cpu : usr=0.21%, sys=3.35%, ctx=5146, majf=0, minf=20 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (usec): 500=1.52%, 750=0.08%, 1000=1.91% lat (msec): 2=71.19%, 4=19.57%, 10=0.51%, 20=2.46%, 50=2.58% lat (msec): 100=0.14%, 250=0.04% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=352MB/s, minb=360MB/s, maxb=360MB/s, mint=14552msec, maxt=14552msec Disk stats (read/write): sdc: ios=0/40837, merge=0/0, ticks=0/63984, in_queue=64172, util=98.64% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=179967098880 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/80M /s] [0/77 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=26616 write: io=7,285MB, bw=121MB/s, iops=121, runt= 60154msec clat (usec): min=425, max=563K, avg=8252.97, stdev=37805.37 bw (KB/s) : min=34240, max=560120, per=107.22%, avg=132971.41, stdev=112411.24 cpu : usr=0.06%, sys=1.15%, ctx=7326, majf=0, minf=48 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/7285, short=0/0 lat (usec): 500=1.07%, 750=0.05%, 1000=3.12% lat (msec): 2=71.43%, 4=18.30%, 10=0.08%, 20=0.82%, 50=2.25% lat (msec): 100=0.33%, 250=1.77%, 500=0.75%, 750=0.01% Run status group 0 (all jobs): WRITE: io=7,285MB, aggrb=121MB/s, minb=124MB/s, maxb=124MB/s, mint=60154msec, maxt=60154msec Disk stats (read/write): sdc: ios=0/58261, merge=0/0, ticks=0/250779, in_queue=250787, util=99.81% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=512 --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/24M /s] [0/47K iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26622 write: io=1,409MB, bw=24,049KB/s, iops=48,097, runt= 60001msec clat (usec): min=22, max=2,487, avg=81.00, stdev=18.09 bw (KB/s) : min= 5466, max= 6612, per=25.01%, avg=6013.92, stdev=105.30 cpu : usr=3.56%, sys=8.88%, ctx=3230180, majf=0, minf=102 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/2885900, short=0/0 lat (usec): 50=28.02%, 100=45.33%, 250=26.58%, 500=0.06%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): WRITE: io=1,409MB, aggrb=24,048KB/s, minb=24,625KB/s, maxb=24,625KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=0/2646284, merge=0/235567, ticks=0/169169, in_queue=169649, util=99.99% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=512 --size=17996709888 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/24M /s] [0/49K iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26630 write: io=1,422MB, bw=24,265KB/s, iops=48,529, runt= 60001msec clat (usec): min=21, max=2,617, avg=80.29, stdev=17.83 bw (KB/s) : min= 5470, max= 6581, per=24.99%, avg=6064.76, stdev=99.60 cpu : usr=3.68%, sys=8.83%, ctx=3271121, majf=0, minf=113 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/2911797, short=0/0 lat (usec): 50=28.18%, 100=46.51%, 250=25.23%, 500=0.06%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01% Run status group 0 (all jobs): WRITE: io=1,422MB, aggrb=24,264KB/s, minb=24,846KB/s, maxb=24,846KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=0/2639923, merge=0/267023, ticks=0/164284, in_queue=164525, util=99.95% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=512 --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [98.4% done] [0K/1,461K /s] [0/3K iops] [eta 00m:01s] file1: (groupid=0, jobs=4): err= 0: pid=26638 write: io=86,538KB, bw=1,442KB/s, iops=2,884, runt= 60001msec clat (usec): min=28, max=9,170, avg=1389.76, stdev=207.20 bw (KB/s) : min= 323, max= 1302, per=24.88%, avg=358.84, stdev=10.81 cpu : usr=0.37%, sys=0.55%, ctx=173197, majf=0, minf=4129 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/173076, short=0/0 lat (usec): 50=0.52%, 100=0.01%, 250=0.01%, 500=0.10%, 750=8.25% lat (usec): 1000=9.38% lat (msec): 2=73.26%, 4=8.45%, 10=0.02% Run status group 0 (all jobs): WRITE: io=86,538KB, aggrb=1,442KB/s, minb=1,476KB/s, maxb=1,476KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=0/172358, merge=0/0, ticks=0/234943, in_queue=234973, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=512 --size=17996709888 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [98.4% done] [0K/1,168K /s] [0/2K iops] [eta 00m:01s] file1: (groupid=0, jobs=4): err= 0: pid=26677 write: io=82,413KB, bw=1,374KB/s, iops=2,747, runt= 60001msec clat (usec): min=29, max=520K, avg=1457.97, stdev=1965.00 bw (KB/s) : min= 199, max= 1313, per=24.98%, avg=342.97, stdev=17.21 cpu : usr=0.35%, sys=0.55%, ctx=165512, majf=0, minf=12122 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/164825, short=0/0 lat (usec): 50=0.71%, 100=2.77%, 250=5.40%, 500=1.67%, 750=7.39% lat (usec): 1000=8.45% lat (msec): 2=61.39%, 4=11.53%, 10=0.45%, 20=0.06%, 50=0.08% lat (msec): 100=0.07%, 250=0.02%, 500=0.01%, 750=0.01% Run status group 0 (all jobs): WRITE: io=82,412KB, aggrb=1,373KB/s, minb=1,406KB/s, maxb=1,406KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdc: ios=0/164289, merge=0/0, ticks=0/235157, in_queue=235186, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/859M /s] [0/838 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26685 write: io=20,480MB, bw=1,294MB/s, iops=1,293, runt= 15827msec clat (usec): min=475, max=89,855, avg=3088.21, stdev=2538.62 bw (KB/s) : min=198193, max=472143, per=25.13%, avg=333015.20, stdev=47852.85 cpu : usr=0.15%, sys=2.27%, ctx=20560, majf=0, minf=92 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (usec): 500=0.01%, 750=0.01%, 1000=0.10% lat (msec): 2=19.59%, 4=78.28%, 10=0.08%, 20=0.15%, 50=1.50% lat (msec): 100=0.27% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=1,294MB/s, minb=1,325MB/s, maxb=1,325MB/s, mint=15827msec, maxt=15827msec Disk stats (read/write): sdc: ios=0/162981, merge=0/0, ticks=0/442123, in_queue=442206, util=99.64% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/1,685M /s] [0/2K iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26691 write: io=20,480MB, bw=1,651MB/s, iops=1,650, runt= 12407msec clat (usec): min=578, max=3,846, avg=2420.11, stdev=235.80 bw (KB/s) : min=414087, max=477184, per=25.03%, avg=423127.04, stdev=5513.64 cpu : usr=0.07%, sys=2.56%, ctx=20547, majf=0, minf=88 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (usec): 750=0.01%, 1000=0.11% lat (msec): 2=20.00%, 4=79.88% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=1,651MB/s, minb=1,690MB/s, maxb=1,690MB/s, mint=12407msec, maxt=12407msec Disk stats (read/write): sdc: ios=0/161168, merge=0/0, ticks=0/340963, in_queue=340995, util=99.45% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/834M /s] [0/814 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26699 write: io=78,763MB, bw=1,313MB/s, iops=1,312, runt= 60002msec clat (usec): min=475, max=108K, avg=3044.25, stdev=2355.74 bw (KB/s) : min=173233, max=471040, per=25.15%, avg=338063.97, stdev=42635.00 cpu : usr=0.08%, sys=1.97%, ctx=79021, majf=0, minf=104 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/78763, short=0/0 lat (usec): 500=0.01%, 750=0.01%, 1000=0.03% lat (msec): 2=19.11%, 4=78.62%, 10=0.24%, 20=0.51%, 50=1.23% lat (msec): 100=0.26%, 250=0.01% Run status group 0 (all jobs): WRITE: io=78,763MB, aggrb=1,313MB/s, minb=1,344MB/s, maxb=1,344MB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdc: ios=0/629512, merge=0/0, ticks=0/1684186, in_queue=1684298, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [100.0% done] [0K/256M /s] [0/250 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26707 write: io=16,385MB, bw=273MB/s, iops=272, runt= 60028msec clat (usec): min=483, max=261K, avg=14648.68, stdev=10435.25 bw (KB/s) : min=41982, max=159244, per=25.07%, avg=70065.30, stdev=7535.41 cpu : usr=0.02%, sys=0.45%, ctx=16448, majf=0, minf=104 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/16385, short=0/0 lat (usec): 500=0.01%, 750=0.01%, 1000=0.03% lat (msec): 2=0.38%, 4=1.27%, 10=78.11%, 20=2.85%, 50=9.52% lat (msec): 100=6.75%, 250=1.06%, 500=0.01% Run status group 0 (all jobs): WRITE: io=16,385MB, aggrb=273MB/s, minb=280MB/s, maxb=280MB/s, mint=60028msec, maxt=60028msec Disk stats (read/write): sdc: ios=0/130718, merge=0/0, ticks=0/1678163, in_queue=1679505, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [100.0% done] [0K/255M /s] [0/249 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=26715 write: io=16,187MB, bw=270MB/s, iops=269, runt= 60006msec clat (usec): min=530, max=230K, avg=14851.39, stdev=10651.68 bw (KB/s) : min=35896, max=142673, per=25.10%, avg=69341.65, stdev=7096.94 cpu : usr=0.03%, sys=0.46%, ctx=16263, majf=0, minf=112 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/16187, short=0/0 lat (usec): 750=0.01%, 1000=0.04% lat (msec): 2=0.45%, 4=1.56%, 10=77.27%, 20=2.61%, 50=10.11% lat (msec): 100=6.82%, 250=1.12% Run status group 0 (all jobs): WRITE: io=16,187MB, aggrb=270MB/s, minb=276MB/s, maxb=276MB/s, mint=60006msec, maxt=60006msec Disk stats (read/write): sdc: ios=0/129292, merge=0/0, ticks=0/1680506, in_queue=1681295, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=179967098880 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [1.2% done] [0K/66M /s] [0/64 iops] [eta 01h:25m:55s] file1: (groupid=0, jobs=4): err= 0: pid=26725 write: io=8,035MB, bw=133MB/s, iops=133, runt= 60341msec clat (usec): min=670, max=474K, avg=30032.47, stdev=33151.37 bw (KB/s) : min= 9394, max=146003, per=26.59%, avg=36260.68, stdev=14578.28 cpu : usr=0.02%, sys=0.23%, ctx=8070, majf=0, minf=164 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/8035, short=0/0 lat (usec): 750=0.01%, 1000=0.10% lat (msec): 2=0.58%, 4=1.54%, 10=73.69%, 20=1.13%, 50=12.82% lat (msec): 100=1.08%, 250=6.30%, 500=2.74% Run status group 0 (all jobs): WRITE: io=8,035MB, aggrb=133MB/s, minb=136MB/s, maxb=136MB/s, mint=60341msec, maxt=60341msec Disk stats (read/write): sdc: ios=0/64253, merge=0/0, ticks=0/1689275, in_queue=1694215, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=512 --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/28M /s] [0/56K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26734 write: io=1,767MB, bw=30,152KB/s, iops=60,303, runt= 60003msec clat (usec): min=23, max=5,905, avg=1060.09, stdev=53.69 bw (KB/s) : min= 418, max= 754, per=1.56%, avg=471.16, stdev= 4.13 cpu : usr=0.30%, sys=0.79%, ctx=4266889, majf=0, minf=1910 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/3618364, short=0/0 lat (usec): 50=0.05%, 100=0.34%, 250=2.20%, 500=8.26%, 750=14.37% lat (usec): 1000=20.66% lat (msec): 2=52.86%, 4=1.27%, 10=0.01% Run status group 0 (all jobs): WRITE: io=1,767MB, aggrb=30,151KB/s, minb=30,875KB/s, maxb=30,875KB/s, mint=60003msec, maxt=60003msec Disk stats (read/write): sdc: ios=0/2538593, merge=0/1073883, ticks=0/2719686, in_queue=2720021, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=512 --size=17996709888 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/30M /s] [0/60K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26836 write: io=1,812MB, bw=30,929KB/s, iops=61,857, runt= 60003msec clat (usec): min=25, max=5,393, avg=1031.59, stdev=55.30 bw (KB/s) : min= 437, max= 708, per=1.57%, avg=484.11, stdev= 3.09 cpu : usr=0.31%, sys=0.81%, ctx=4403398, majf=0, minf=1783 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/3711610, short=0/0 lat (usec): 50=0.06%, 100=0.38%, 250=2.40%, 500=8.80%, 750=14.98% lat (usec): 1000=21.64% lat (msec): 2=50.85%, 4=0.88%, 10=0.01% Run status group 0 (all jobs): WRITE: io=1,812MB, aggrb=30,928KB/s, minb=31,670KB/s, maxb=31,670KB/s, mint=60003msec, maxt=60003msec Disk stats (read/write): sdc: ios=0/2545541, merge=0/1160180, ticks=0/2648520, in_queue=2648735, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=512 --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 7 (f=7): [______________ww__ww______www___________________________________] [1.8% done] [0K/1,469K /s] [0/3K iops] [eta 57m:00s]s] file1: (groupid=0, jobs=64): err= 0: pid=26904 write: io=85,691KB, bw=1,422KB/s, iops=2,843, runt= 60274msec clat (usec): min=28, max=406K, avg=23071.33, stdev=1179.63 bw (KB/s) : min= 5, max= 1177, per=1.48%, avg=21.09, stdev= 0.35 cpu : usr=0.03%, sys=0.04%, ctx=171655, majf=0, minf=10672 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/171381, short=0/0 lat (usec): 50=0.52%, 100=0.01%, 250=0.01%, 500=0.08%, 750=0.16% lat (usec): 1000=0.05% lat (msec): 2=0.33%, 4=0.72%, 10=2.12%, 20=3.39%, 50=92.45% lat (msec): 100=0.09%, 500=0.07% Run status group 0 (all jobs): WRITE: io=85,690KB, aggrb=1,421KB/s, minb=1,455KB/s, maxb=1,455KB/s, mint=60274msec, maxt=60274msec Disk stats (read/write): sdc: ios=0/170663, merge=0/0, ticks=0/3835416, in_queue=3835430, util=99.99% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=512 --size=17996709888 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 10 (f=10): [_______w__w_______wwwww________w___w_______________________w____] [1.9% done] [0K/1,163K /s] [0/2K iops] [eta 54m:00s] file1: (groupid=0, jobs=64): err= 0: pid=26972 write: io=79,981KB, bw=1,329KB/s, iops=2,657, runt= 60203msec clat (usec): min=23, max=311K, avg=24729.98, stdev=2773.76 bw (KB/s) : min= 8, max= 1179, per=1.48%, avg=19.63, stdev= 0.42 cpu : usr=0.04%, sys=0.04%, ctx=163010, majf=0, minf=40040 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/159961, short=0/0 lat (usec): 50=0.59%, 100=0.07%, 250=0.10%, 500=0.19%, 750=0.24% lat (usec): 1000=0.20% lat (msec): 2=1.77%, 4=7.48%, 10=6.42%, 20=5.67%, 50=72.66% lat (msec): 100=2.38%, 250=2.15%, 500=0.07% Run status group 0 (all jobs): WRITE: io=79,980KB, aggrb=1,328KB/s, minb=1,360KB/s, maxb=1,360KB/s, mint=60203msec, maxt=60203msec Disk stats (read/write): sdc: ios=0/159876, merge=0/1, ticks=0/3837387, in_queue=3837413, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/2,466M /s] [0/2K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27042 write: io=141GB, bw=2,406MB/s, iops=2,405, runt= 60026msec clat (usec): min=671, max=187K, avg=26640.08, stdev=3437.23 bw (KB/s) : min=30062, max=53035, per=1.56%, avg=38457.94, stdev=288.51 cpu : usr=0.01%, sys=0.31%, ctx=189593, majf=0, minf=1846 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/144419, short=0/0 lat (usec): 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=66.31%, 20=3.58%, 50=0.04% lat (msec): 100=30.07%, 250=0.01% Run status group 0 (all jobs): WRITE: io=141GB, aggrb=2,406MB/s, minb=2,464MB/s, maxb=2,464MB/s, mint=60026msec, maxt=60026msec Disk stats (read/write): sdc: ios=0/1152816, merge=0/0, ticks=0/7891448, in_queue=7892149, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/2,510M /s] [0/2K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27110 write: io=143GB, bw=2,446MB/s, iops=2,445, runt= 60028msec clat (usec): min=521, max=183K, avg=25944.21, stdev=3458.30 bw (KB/s) : min=28550, max=55138, per=1.58%, avg=39469.65, stdev=276.88 cpu : usr=0.01%, sys=0.31%, ctx=191386, majf=0, minf=1589 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/146807, short=0/0 lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=68.55%, 20=2.13%, 50=0.03% lat (msec): 100=29.27%, 250=0.01% Run status group 0 (all jobs): WRITE: io=143GB, aggrb=2,446MB/s, minb=2,504MB/s, maxb=2,504MB/s, mint=60028msec, maxt=60028msec Disk stats (read/write): sdc: ios=0/1171961, merge=0/0, ticks=0/7888648, in_queue=7889176, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=write --bs=1m --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/2,494M /s] [0/2K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27178 write: io=143GB, bw=2,435MB/s, iops=2,435, runt= 60029msec clat (usec): min=619, max=174K, avg=26438.33, stdev=3477.00 bw (KB/s) : min=28764, max=51200, per=1.55%, avg=38737.18, stdev=301.39 cpu : usr=0.01%, sys=0.31%, ctx=190754, majf=0, minf=1909 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/146193, short=0/0 lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=68.21%, 20=2.33%, 50=0.03% lat (msec): 100=29.40%, 250=0.01% Run status group 0 (all jobs): WRITE: io=143GB, aggrb=2,435MB/s, minb=2,494MB/s, maxb=2,494MB/s, mint=60029msec, maxt=60029msec Disk stats (read/write): sdc: ios=0/1167104, merge=0/0, ticks=0/7888575, in_queue=7889122, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 38 (f=38): [wEwE_wwww__ww_wwww_E__wwwww__w_wEww__w__www_w_w_www_wwwwwE_w_wwE] [4.8% done] [0K/187M /s] [0/182 iops] [eta 19m:59s] file1: (groupid=0, jobs=64): err= 0: pid=27246 write: io=14,406MB, bw=239MB/s, iops=238, runt= 60364msec clat (usec): min=627, max=673K, avg=267334.48, stdev=11077.10 bw (KB/s) : min= 1582, max=15142, per=1.59%, avg=3893.28, stdev=138.30 cpu : usr=0.00%, sys=0.03%, ctx=28452, majf=0, minf=1459 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/14406, short=0/0 lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.06%, 10=0.48%, 20=0.22%, 50=0.75% lat (msec): 100=0.62%, 250=37.89%, 500=59.61%, 750=0.34% Run status group 0 (all jobs): WRITE: io=14,406MB, aggrb=239MB/s, minb=244MB/s, maxb=244MB/s, mint=60364msec, maxt=60364msec Disk stats (read/write): sdc: ios=0/114948, merge=0/0, ticks=0/7932876, in_queue=7936690, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=174598389760 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 19 (f=19): [______ww__w_w_Ewww_w__w_______w__w__w__ww______w__w__w____w___w_] [4.6% done] [0K/250M /s] [0/243 iops] [eta 21m:08s] file1: (groupid=0, jobs=64): err= 0: pid=27316 write: io=15,303MB, bw=254MB/s, iops=253, runt= 60293msec clat (usec): min=656, max=615K, avg=251042.98, stdev=9332.08 bw (KB/s) : min= 1774, max=15673, per=1.59%, avg=4126.38, stdev=121.88 cpu : usr=0.00%, sys=0.03%, ctx=30288, majf=0, minf=2114 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/15303, short=0/0 lat (usec): 750=0.01%, 1000=0.01% lat (msec): 2=0.02%, 4=0.07%, 10=0.46%, 20=0.18%, 50=0.66% lat (msec): 100=0.59%, 250=45.10%, 500=52.59%, 750=0.31% Run status group 0 (all jobs): WRITE: io=15,303MB, aggrb=254MB/s, minb=260MB/s, maxb=260MB/s, mint=60293msec, maxt=60293msec Disk stats (read/write): sdc: ios=0/122275, merge=0/0, ticks=0/7926674, in_queue=7928781, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/sdc --direct=1 --rw=randwrite --bs=1m --size=179967098880 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 24 (f=24): [_____EEw___w_w____w_wE__www__ww_E_w__w__wEE_w_w__w__w___wwwEwwww] [3.0% done] [0K/38M /s] [0/36 iops] [eta 32m:59s] file1: (groupid=0, jobs=64): err= 0: pid=27384 write: io=7,159MB, bw=117MB/s, iops=116, runt= 61428msec clat (usec): min=627, max=2,161K, avg=543600.75, stdev=56061.27 bw (KB/s) : min= 473, max=13837, per=1.98%, avg=2358.09, stdev=256.24 cpu : usr=0.00%, sys=0.01%, ctx=14130, majf=0, minf=1459 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/7159, short=0/0 lat (usec): 750=0.03%, 1000=0.04% lat (msec): 2=0.07%, 4=0.07%, 10=1.10%, 20=0.28%, 50=0.88% lat (msec): 100=0.38%, 250=51.26%, 500=3.63%, 750=7.64%, 1000=12.85% lat (msec): 2000=21.75%, >=2000=0.01% Run status group 0 (all jobs): WRITE: io=7,159MB, aggrb=117MB/s, minb=119MB/s, maxb=119MB/s, mint=61428msec, maxt=61428msec Disk stats (read/write): sdc: ios=0/57151, merge=0/0, ticks=0/8066227, in_queue=8096893, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ [root@iotest ~]#