[root@iotest ~]# cat fiotester2.sh #!/bin/bash FIO="/usr/bin/fio" FN="/dev/fioa" OFFSET="317184475136" # 5GB are 5.368.709.120 Bytes DEVSIZE="322553184256" DEVSIZE10P="32255318425" BSIOPS="512" BSTHR="1m" for TESTCASE in "read" "write" do for NJ in 1 4 64 do echo "#################################################################################" echo "Executing $TESTCASE tests with numjobs=$NJ" for testrun in `seq 1 10` do case $testrun in 1) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 2) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 3) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 4) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSIOPS --size=$DEVSIZE10P --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 5) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 6) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";; 7) TESTCMD="$FIO --filename=$FN --direct=1 --rw=$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 8) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; 9) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=5G --runtime=60 --offset=$OFFSET --numjobs=$NJ --group_reporting --name=file1";; 10) TESTCMD="$FIO --filename=$FN --direct=1 --rw=rand$TESTCASE --bs=$BSTHR --size=$DEVSIZE --runtime=60 --numjobs=$NJ --group_reporting --name=file1";; esac echo "---------------------------------------------------------------------------------" echo "Executing test $testrun: $TESTCMD" $TESTCMD echo "Sleeping for 30 seconds... ------------------------------------------------------" sleep 30 done done done [root@iotest ~]# ./fiotester2.sh ################################################################################# Executing read tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=512 --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [12M/0K /s] [23K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27486 read : io=697MB, bw=11,889KB/s, iops=23,777, runt= 60001msec clat (usec): min=27, max=5,933, avg=40.10, stdev=53.81 bw (KB/s) : min= 904, max=13714, per=100.03%, avg=11891.11, stdev=1625.68 cpu : usr=6.57%, sys=17.39%, ctx=1427188, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1426644/0, short=0/0 lat (usec): 50=96.85%, 100=2.87%, 250=0.19%, 500=0.01%, 750=0.01% lat (usec): 1000=0.04% lat (msec): 2=0.02%, 4=0.02%, 10=0.01% Run status group 0 (all jobs): READ: io=697MB, aggrb=11,888KB/s, minb=12,173KB/s, maxb=12,173KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=1424396/0, merge=0/0, ticks=42727/0, in_queue=42613, util=71.28% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=512 --size=32255318425 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [12M/0K /s] [23K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27491 read : io=672MB, bw=11,471KB/s, iops=22,942, runt= 60001msec clat (usec): min=31, max=6,389, avg=41.68, stdev=57.57 bw (KB/s) : min= 236, max=12537, per=100.00%, avg=11471.17, stdev=1546.76 cpu : usr=4.87%, sys=17.67%, ctx=1377041, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1376574/0, short=0/0 lat (usec): 50=94.85%, 100=4.86%, 250=0.20%, 500=0.01%, 750=0.01% lat (usec): 1000=0.05% lat (msec): 2=0.02%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): READ: io=672MB, aggrb=11,471KB/s, minb=11,746KB/s, maxb=11,746KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=1374265/0, merge=0/0, ticks=49512/0, in_queue=49148, util=82.21% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=512 --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [6,493K/0K /s] [13K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27496 read : io=360MB, bw=6,147KB/s, iops=12,293, runt= 60001msec clat (usec): min=36, max=6,792, avg=78.23, stdev=79.24 bw (KB/s) : min= 819, max= 6667, per=99.99%, avg=6145.25, stdev=775.64 cpu : usr=4.61%, sys=9.62%, ctx=737993, majf=0, minf=4153 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=737633/0, short=0/0 lat (usec): 50=0.01%, 100=97.41%, 250=2.42%, 500=0.01%, 750=0.01% lat (usec): 1000=0.08% lat (msec): 2=0.03%, 4=0.03%, 10=0.01% Run status group 0 (all jobs): READ: io=360MB, aggrb=6,146KB/s, minb=6,294KB/s, maxb=6,294KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=735690/0, merge=0/0, ticks=46496/0, in_queue=46665, util=78.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=512 --size=32255318425 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [6,215K/0K /s] [12K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27501 read : io=349MB, bw=5,954KB/s, iops=11,907, runt= 60001msec clat (usec): min=34, max=6,051, avg=81.03, stdev=79.39 bw (KB/s) : min= 274, max= 6280, per=100.01%, avg=5953.79, stdev=755.93 cpu : usr=3.44%, sys=9.32%, ctx=714727, majf=0, minf=2079 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=714440/0, short=0/0 lat (usec): 50=0.01%, 100=97.18%, 250=2.64%, 500=0.01%, 750=0.02% lat (usec): 1000=0.09% lat (msec): 2=0.03%, 4=0.02%, 10=0.01% Run status group 0 (all jobs): READ: io=349MB, aggrb=5,953KB/s, minb=6,096KB/s, maxb=6,096KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=712590/0, merge=0/0, ticks=50961/0, in_queue=50800, util=84.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [700M/0K /s] [683/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27508 read : io=5,120MB, bw=682MB/s, iops=682, runt= 7502msec clat (usec): min=1,402, max=2,393, avg=1462.06, stdev=88.97 bw (KB/s) : min=690842, max=704512, per=100.10%, avg=699531.71, stdev=3656.61 cpu : usr=0.47%, sys=7.41%, ctx=5144, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=98.87%, 4=1.13% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=682MB/s, minb=699MB/s, maxb=699MB/s, mint=7502msec, maxt=7502msec Disk stats (read/write): fioa: ios=40592/0, merge=0/0, ticks=32658/0, in_queue=32667, util=94.74% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [696M/0K /s] [679/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27511 read : io=5,120MB, bw=631MB/s, iops=631, runt= 8110msec clat (usec): min=1,386, max=7,381, avg=1580.96, stdev=661.97 bw (KB/s) : min=214938, max=708608, per=100.08%, avg=647001.44, stdev=134049.84 cpu : usr=0.30%, sys=6.44%, ctx=5145, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=95.25%, 4=2.58%, 10=2.17% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=631MB/s, minb=646MB/s, maxb=646MB/s, mint=8110msec, maxt=8110msec Disk stats (read/write): fioa: ios=39977/0, merge=0/0, ticks=36868/0, in_queue=36875, util=95.26% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [R] [100.0% done] [685M/0K /s] [669/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27516 read : io=39,502MB, bw=658MB/s, iops=658, runt= 60002msec clat (usec): min=1,405, max=7,674, avg=1515.94, stdev=350.29 bw (KB/s) : min=216966, max=700416, per=100.10%, avg=674814.99, stdev=74364.56 cpu : usr=0.39%, sys=9.67%, ctx=39834, majf=0, minf=283 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=39502/0, short=0/0 lat (msec): 2=98.21%, 4=1.25%, 10=0.54% Run status group 0 (all jobs): READ: io=39,502MB, aggrb=658MB/s, minb=674MB/s, maxb=674MB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): fioa: ios=315506/0, merge=0/0, ticks=264528/0, in_queue=264530, util=94.10% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [682M/0K /s] [666/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27521 read : io=5,120MB, bw=665MB/s, iops=664, runt= 7703msec clat (usec): min=1,420, max=2,495, avg=1498.91, stdev=85.39 bw (KB/s) : min=679936, max=684032, per=100.11%, avg=681347.47, stdev=1222.53 cpu : usr=0.18%, sys=3.87%, ctx=5142, majf=0, minf=277 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=98.95%, 4=1.05% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=665MB/s, minb=681MB/s, maxb=681MB/s, mint=7703msec, maxt=7703msec Disk stats (read/write): fioa: ios=40868/0, merge=0/0, ticks=33598/0, in_queue=33602, util=93.41% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [703M/0K /s] [686/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27524 read : io=5,120MB, bw=685MB/s, iops=685, runt= 7473msec clat (usec): min=1,385, max=2,373, avg=1454.86, stdev=84.05 bw (KB/s) : min=700416, max=704512, per=100.09%, avg=702200.50, stdev=1127.38 cpu : usr=0.25%, sys=6.81%, ctx=5145, majf=0, minf=277 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=98.98%, 4=1.02% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=685MB/s, minb=702MB/s, maxb=702MB/s, mint=7473msec, maxt=7473msec Disk stats (read/write): fioa: ios=40713/0, merge=0/0, ticks=32601/0, in_queue=32616, util=95.22% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [r] [100.0% done] [714M/0K /s] [696/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=27529 read : io=40,897MB, bw=682MB/s, iops=681, runt= 60001msec clat (usec): min=1,376, max=8,261, avg=1462.69, stdev=349.75 bw (KB/s) : min=212992, max=718848, per=100.07%, avg=698479.64, stdev=79416.07 cpu : usr=0.35%, sys=7.21%, ctx=41125, majf=0, minf=343 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=40897/0, short=0/0 lat (msec): 2=98.22%, 4=1.25%, 10=0.54% Run status group 0 (all jobs): READ: io=40,897MB, aggrb=682MB/s, minb=698MB/s, maxb=698MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=326651/0, merge=0/0, ticks=265301/0, in_queue=265306, util=96.22% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=512 --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [25M/0K /s] [50K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27538 read : io=1,675MB, bw=28,595KB/s, iops=57,189, runt= 60001msec clat (usec): min=24, max=6,879, avg=68.10, stdev=42.60 bw (KB/s) : min= 301, max=10188, per=25.05%, avg=7161.40, stdev=1050.30 cpu : usr=3.36%, sys=9.79%, ctx=3433362, majf=0, minf=98 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=3431406/0, short=0/0 lat (usec): 50=47.46%, 100=37.15%, 250=15.26%, 500=0.01%, 750=0.01% lat (usec): 1000=0.06% lat (msec): 2=0.02%, 4=0.03%, 10=0.02% Run status group 0 (all jobs): READ: io=1,675MB, aggrb=28,594KB/s, minb=29,280KB/s, maxb=29,280KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=3426737/0, merge=1346/0, ticks=203603/0, in_queue=202695, util=96.96% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=512 --size=32255318425 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [21M/0K /s] [43K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27575 read : io=1,280MB, bw=21,838KB/s, iops=43,676, runt= 60001msec clat (usec): min=25, max=7,628, avg=89.66, stdev=51.40 bw (KB/s) : min= 140, max=10865, per=25.00%, avg=5458.87, stdev=668.73 cpu : usr=3.30%, sys=7.95%, ctx=2622972, majf=0, minf=108 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=2620642/0, short=0/0 lat (usec): 50=15.96%, 100=52.66%, 250=30.28%, 500=0.93%, 750=0.01% lat (usec): 1000=0.07% lat (msec): 2=0.03%, 4=0.03%, 10=0.02% Run status group 0 (all jobs): READ: io=1,280MB, aggrb=21,838KB/s, minb=22,362KB/s, maxb=22,362KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=2616935/0, merge=10/0, ticks=205437/0, in_queue=205277, util=99.70% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=512 --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 2 (f=2): [_r_r] [33.7% done] [19M/0K /s] [39K/0 iops] [eta 02m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27583 read : io=1,119MB, bw=19,096KB/s, iops=38,191, runt= 60001msec clat (usec): min=44, max=6,621, avg=101.10, stdev=52.97 bw (KB/s) : min= 146, max= 5235, per=25.18%, avg=4807.75, stdev=374.59 cpu : usr=3.23%, sys=7.36%, ctx=2299013, majf=0, minf=4210 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=2291505/0, short=0/0 lat (usec): 50=0.01%, 100=61.89%, 250=37.64%, 500=0.28%, 750=0.01% lat (usec): 1000=0.08% lat (msec): 2=0.03%, 4=0.04%, 10=0.03% Run status group 0 (all jobs): READ: io=1,119MB, aggrb=19,095KB/s, minb=19,553KB/s, maxb=19,553KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=2290213/0, merge=0/0, ticks=212691/0, in_queue=212963, util=99.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=512 --size=32255318425 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 2 (f=2): [__rr] [33.7% done] [19M/0K /s] [39K/0 iops] [eta 02m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27593 read : io=1,090MB, bw=18,610KB/s, iops=37,219, runt= 60001msec clat (usec): min=36, max=6,828, avg=105.31, stdev=54.67 bw (KB/s) : min= 140, max= 5133, per=24.83%, avg=4619.74, stdev=440.35 cpu : usr=3.10%, sys=7.59%, ctx=2238460, majf=0, minf=13451 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=2233232/0, short=0/0 lat (usec): 50=0.01%, 100=60.05%, 250=38.66%, 500=1.09%, 750=0.01% lat (usec): 1000=0.08% lat (msec): 2=0.03%, 4=0.04%, 10=0.03% Run status group 0 (all jobs): READ: io=1,090MB, aggrb=18,609KB/s, minb=19,056KB/s, maxb=19,056KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=2232094/0, merge=0/0, ticks=213024/0, in_queue=213137, util=99.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [683M/0K /s] [666/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27601 read : io=20,480MB, bw=729MB/s, iops=729, runt= 28087msec clat (usec): min=1,717, max=15,107, avg=5482.45, stdev=581.51 bw (KB/s) : min=142788, max=196215, per=25.03%, avg=186896.47, stdev=6891.53 cpu : usr=0.07%, sys=2.06%, ctx=20623, majf=0, minf=1116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=0.03%, 4=5.38%, 10=93.52%, 20=1.07% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=729MB/s, minb=747MB/s, maxb=747MB/s, mint=28087msec, maxt=28087msec Disk stats (read/write): fioa: ios=163424/0, merge=0/0, ticks=627277/0, in_queue=627352, util=99.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [771M/0K /s] [752/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27609 read : io=20,480MB, bw=750MB/s, iops=749, runt= 27319msec clat (usec): min=1,471, max=13,273, avg=5330.28, stdev=427.81 bw (KB/s) : min=148023, max=194661, per=25.03%, avg=192129.57, stdev=2966.12 cpu : usr=0.09%, sys=2.22%, ctx=20647, majf=0, minf=1116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=0.01%, 4=5.86%, 10=93.96%, 20=0.17% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=750MB/s, minb=768MB/s, maxb=768MB/s, mint=27319msec, maxt=27319msec Disk stats (read/write): fioa: ios=163505/0, merge=0/0, ticks=608820/0, in_queue=608858, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [RRRR] [100.0% done] [771M/0K /s] [752/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27615 read : io=43,336MB, bw=722MB/s, iops=722, runt= 60005msec clat (usec): min=1,469, max=14,640, avg=5536.63, stdev=544.24 bw (KB/s) : min=143073, max=196608, per=25.01%, avg=184990.90, stdev=6303.18 cpu : usr=0.10%, sys=2.10%, ctx=43691, majf=0, minf=1124 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=43336/0, short=0/0 lat (msec): 2=0.01%, 4=4.38%, 10=94.59%, 20=1.03% Run status group 0 (all jobs): READ: io=43,336MB, aggrb=722MB/s, minb=740MB/s, maxb=740MB/s, mint=60005msec, maxt=60005msec Disk stats (read/write): fioa: ios=346053/0, merge=0/0, ticks=1355650/0, in_queue=1355704, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [730M/0K /s] [712/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27625 read : io=20,480MB, bw=722MB/s, iops=722, runt= 28354msec clat (usec): min=2,141, max=14,220, avg=5531.71, stdev=544.85 bw (KB/s) : min=155826, max=194661, per=25.03%, avg=185128.34, stdev=6175.77 cpu : usr=0.14%, sys=1.85%, ctx=20604, majf=0, minf=1108 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 4=4.38%, 10=94.65%, 20=0.97% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=722MB/s, minb=740MB/s, maxb=740MB/s, mint=28354msec, maxt=28354msec Disk stats (read/write): fioa: ios=163379/0, merge=0/0, ticks=641484/0, in_queue=641532, util=99.90% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [745M/0K /s] [727/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27631 read : io=20,480MB, bw=750MB/s, iops=750, runt= 27301msec clat (usec): min=2,180, max=13,454, avg=5324.69, stdev=423.15 bw (KB/s) : min=168300, max=194560, per=25.05%, avg=192444.00, stdev=1583.02 cpu : usr=0.11%, sys=2.09%, ctx=20616, majf=0, minf=1108 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 4=5.79%, 10=94.11%, 20=0.11% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=750MB/s, minb=768MB/s, maxb=768MB/s, mint=27301msec, maxt=27301msec Disk stats (read/write): fioa: ios=163675/0, merge=0/0, ticks=611087/0, in_queue=611134, util=99.87% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [rrrr] [100.0% done] [773M/0K /s] [754/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=27639 read : io=44,122MB, bw=735MB/s, iops=735, runt= 60004msec clat (usec): min=1,554, max=16,163, avg=5434.15, stdev=526.48 bw (KB/s) : min=153600, max=195047, per=25.03%, avg=188434.20, stdev=5015.41 cpu : usr=0.12%, sys=1.92%, ctx=44450, majf=0, minf=1333 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=44122/0, short=0/0 lat (msec): 2=0.01%, 4=5.13%, 10=94.04%, 20=0.83% Run status group 0 (all jobs): READ: io=44,122MB, aggrb=735MB/s, minb=753MB/s, maxb=753MB/s, mint=60004msec, maxt=60004msec Disk stats (read/write): fioa: ios=352419/0, merge=0/0, ticks=1357414/0, in_queue=1357502, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=512 --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [27M/0K /s] [53K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27677 read : io=1,859MB, bw=31,729KB/s, iops=63,457, runt= 60003msec clat (usec): min=27, max=10,172, avg=964.81, stdev=102.60 bw (KB/s) : min= 167, max= 2120, per=1.63%, avg=516.06, stdev=40.80 cpu : usr=0.36%, sys=1.03%, ctx=3896073, majf=0, minf=1845 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=3807626/0, short=0/0 lat (usec): 50=0.01%, 100=0.03%, 250=6.39%, 500=31.22%, 750=15.65% lat (usec): 1000=8.26% lat (msec): 2=24.21%, 4=14.06%, 10=0.18%, 20=0.01% Run status group 0 (all jobs): READ: io=1,859MB, aggrb=31,728KB/s, minb=32,490KB/s, maxb=32,490KB/s, mint=60003msec, maxt=60003msec Disk stats (read/write): fioa: ios=3801079/0, merge=1198/0, ticks=3759762/0, in_queue=3756787, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=512 --size=32255318425 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [33M/0K /s] [65K/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27745 read : io=2,102MB, bw=35,865KB/s, iops=71,729, runt= 60005msec clat (usec): min=32, max=9,987, avg=916.81, stdev=93.66 bw (KB/s) : min= 167, max= 1975, per=1.52%, avg=546.17, stdev=40.89 cpu : usr=0.40%, sys=1.25%, ctx=4409376, majf=0, minf=1973 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=4304105/0, short=0/0 lat (usec): 50=0.01%, 100=0.03%, 250=8.81%, 500=36.02%, 750=13.41% lat (usec): 1000=8.02% lat (msec): 2=23.67%, 4=9.84%, 10=0.21% Run status group 0 (all jobs): READ: io=2,102MB, aggrb=35,864KB/s, minb=36,725KB/s, maxb=36,725KB/s, mint=60005msec, maxt=60005msec Disk stats (read/write): fioa: ios=4295054/0, merge=1801/0, ticks=3748650/0, in_queue=3744400, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=512 --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 12 (f=12): [___________rrrrrr__________rr________rrr__________r_____________] [2.0% done] [18M/0K /s] [35K/0 iops] [eta 52m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27815 read : io=2,179MB, bw=37,183KB/s, iops=74,366, runt= 60008msec clat (usec): min=52, max=9,644, avg=850.67, stdev=66.12 bw (KB/s) : min= 151, max= 1492, per=1.57%, avg=584.82, stdev=14.48 cpu : usr=0.54%, sys=1.16%, ctx=4602460, majf=0, minf=47404 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=4462579/0, short=0/0 lat (usec): 100=0.18%, 250=1.09%, 500=27.92%, 750=22.40%, 1000=16.36% lat (msec): 2=29.30%, 4=2.63%, 10=0.13% Run status group 0 (all jobs): READ: io=2,179MB, aggrb=37,183KB/s, minb=38,075KB/s, maxb=38,075KB/s, mint=60008msec, maxt=60008msec Disk stats (read/write): fioa: ios=4457740/0, merge=0/0, ticks=3760678/0, in_queue=3758308, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=512 --size=32255318425 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 12 (f=12): [_________r______________r______rrrr______rr__rr_r___________r___] [2.0% done] [16M/0K /s] [32K/0 iops] [eta 52m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27883 read : io=2,065MB, bw=35,238KB/s, iops=70,475, runt= 60004msec clat (usec): min=39, max=10,566, avg=916.72, stdev=68.88 bw (KB/s) : min= 411, max= 4582, per=1.53%, avg=540.65, stdev=11.17 cpu : usr=0.51%, sys=1.12%, ctx=4361408, majf=0, minf=113963 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=4228805/0, short=0/0 lat (usec): 50=0.01%, 100=0.19%, 250=1.23%, 500=25.45%, 750=21.78% lat (usec): 1000=16.37% lat (msec): 2=30.70%, 4=4.15%, 10=0.13%, 20=0.01% Run status group 0 (all jobs): READ: io=2,065MB, aggrb=35,237KB/s, minb=36,083KB/s, maxb=36,083KB/s, mint=60004msec, maxt=60004msec Disk stats (read/write): fioa: ios=4227102/0, merge=0/0, ticks=3761387/0, in_queue=3758991, util=99.97% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [755M/0K /s] [737/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=27951 read : io=45,081MB, bw=751MB/s, iops=750, runt= 60040msec clat (msec): min=7, max=125, avg=85.04, stdev= 1.52 bw (KB/s) : min= 9961, max=15544, per=1.57%, avg=12039.07, stdev=63.66 cpu : usr=0.00%, sys=0.15%, ctx=89665, majf=0, minf=18101 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=45081/0, short=0/0 lat (msec): 10=0.01%, 20=0.01%, 50=1.12%, 100=89.66%, 250=9.21% Run status group 0 (all jobs): READ: io=45,081MB, aggrb=751MB/s, minb=769MB/s, maxb=769MB/s, mint=60040msec, maxt=60040msec Disk stats (read/write): fioa: ios=359823/0, merge=0/0, ticks=7942142/0, in_queue=7946423, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [751M/0K /s] [733/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28019 read : io=45,172MB, bw=752MB/s, iops=752, runt= 60047msec clat (msec): min=2, max=141, avg=85.27, stdev= 1.45 bw (KB/s) : min=10893, max=15226, per=1.56%, avg=12008.98, stdev=51.86 cpu : usr=0.01%, sys=0.15%, ctx=90089, majf=0, minf=18293 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=45172/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=0.02%, 50=1.10%, 100=90.39% lat (msec): 250=8.49% Run status group 0 (all jobs): READ: io=45,172MB, aggrb=752MB/s, minb=770MB/s, maxb=770MB/s, mint=60047msec, maxt=60047msec Disk stats (read/write): fioa: ios=360491/0, merge=0/0, ticks=7941647/0, in_queue=7945083, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=read --bs=1m --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR] [100.0% done] [785M/0K /s] [766/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28089 read : io=45,286MB, bw=754MB/s, iops=754, runt= 60036msec clat (msec): min=5, max=159, avg=84.66, stdev= 1.48 bw (KB/s) : min=10574, max=14336, per=1.57%, avg=12099.10, stdev=64.42 cpu : usr=0.01%, sys=0.15%, ctx=90246, majf=0, minf=18165 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=45286/0, short=0/0 lat (msec): 10=0.01%, 20=0.01%, 50=1.02%, 100=89.95%, 250=9.02% Run status group 0 (all jobs): READ: io=45,286MB, aggrb=754MB/s, minb=772MB/s, maxb=772MB/s, mint=60036msec, maxt=60036msec Disk stats (read/write): fioa: ios=361490/0, merge=0/0, ticks=7948060/0, in_queue=7951997, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr] [100.0% done] [762M/0K /s] [743/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28157 read : io=45,089MB, bw=751MB/s, iops=750, runt= 60067msec clat (usec): min=2,346, max=195K, avg=85588.15, stdev=895.16 bw (KB/s) : min= 9411, max=15541, per=1.56%, avg=11973.53, stdev=43.31 cpu : usr=0.01%, sys=0.15%, ctx=89909, majf=0, minf=18735 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=45089/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=0.01%, 50=1.47%, 100=97.23% lat (msec): 250=1.28% Run status group 0 (all jobs): READ: io=45,089MB, aggrb=751MB/s, minb=769MB/s, maxb=769MB/s, mint=60067msec, maxt=60067msec Disk stats (read/write): fioa: ios=359731/0, merge=0/0, ticks=7944255/0, in_queue=7946550, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr] [100.0% done] [767M/0K /s] [748/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28225 read : io=45,004MB, bw=749MB/s, iops=749, runt= 60076msec clat (msec): min=3, max=108, avg=85.24, stdev= 1.10 bw (KB/s) : min=11150, max=15603, per=1.57%, avg=12014.99, stdev=56.28 cpu : usr=0.01%, sys=0.15%, ctx=90142, majf=0, minf=18182 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=45004/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=0.02%, 50=1.71%, 100=97.01% lat (msec): 250=1.26% Run status group 0 (all jobs): READ: io=45,004MB, aggrb=749MB/s, minb=767MB/s, maxb=767MB/s, mint=60076msec, maxt=60076msec Disk stats (read/write): fioa: ios=358998/0, merge=0/0, ticks=7942074/0, in_queue=7943911, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randread --bs=1m --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr] [100.0% done] [767M/0K /s] [749/0 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28293 read : io=44,981MB, bw=749MB/s, iops=749, runt= 60048msec clat (msec): min=3, max=131, avg=85.47, stdev= 1.03 bw (KB/s) : min=10951, max=15031, per=1.56%, avg=11987.59, stdev=51.85 cpu : usr=0.01%, sys=0.15%, ctx=90635, majf=0, minf=21460 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=44981/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=0.02%, 50=1.47%, 100=97.71% lat (msec): 250=0.79% Run status group 0 (all jobs): READ: io=44,981MB, aggrb=749MB/s, minb=767MB/s, maxb=767MB/s, mint=60048msec, maxt=60048msec Disk stats (read/write): fioa: ios=358919/0, merge=0/0, ticks=7938265/0, in_queue=7941569, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=512 --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/10M /s] [0/21K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28391 write: io=583MB, bw=9,958KB/s, iops=19,916, runt= 60000msec clat (usec): min=35, max=5,421, avg=48.14, stdev=48.93 bw (KB/s) : min= 1911, max=10754, per=99.96%, avg=9953.54, stdev=1015.29 cpu : usr=6.00%, sys=15.42%, ctx=1195489, majf=0, minf=24 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1195004, short=0/0 lat (usec): 50=76.29%, 100=23.31%, 250=0.32%, 500=0.01%, 750=0.01% lat (usec): 1000=0.05% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=583MB, aggrb=9,958KB/s, minb=10,197KB/s, maxb=10,197KB/s, mint=60000msec, maxt=60000msec Disk stats (read/write): fioa: ios=0/1193079, merge=0/0, ticks=0/46016, in_queue=46024, util=76.94% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=512 --size=32255318425 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/11M /s] [0/21K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28398 write: io=609MB, bw=10,389KB/s, iops=20,778, runt= 60001msec clat (usec): min=34, max=4,684, avg=46.09, stdev=40.79 bw (KB/s) : min= 4700, max=11158, per=100.01%, avg=10388.53, stdev=723.77 cpu : usr=7.50%, sys=16.75%, ctx=1247269, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1246701, short=0/0 lat (usec): 50=86.50%, 100=13.13%, 250=0.30%, 500=0.01%, 750=0.01% lat (usec): 1000=0.05% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=609MB, aggrb=10,388KB/s, minb=10,638KB/s, maxb=10,638KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/1244736, merge=0/0, ticks=0/43546, in_queue=43493, util=72.68% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=512 --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/10M /s] [0/20K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28403 write: io=585MB, bw=9,982KB/s, iops=19,964, runt= 60001msec clat (usec): min=35, max=6,198, avg=46.74, stdev=41.99 bw (KB/s) : min= 4484, max=10343, per=100.01%, avg=9982.87, stdev=635.57 cpu : usr=8.23%, sys=15.76%, ctx=1198515, majf=0, minf=2051 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1197903, short=0/0 lat (usec): 50=83.04%, 100=16.60%, 250=0.29%, 500=0.01%, 750=0.01% lat (usec): 1000=0.05% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=585MB, aggrb=9,982KB/s, minb=10,221KB/s, maxb=10,221KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/1195055, merge=0/0, ticks=0/50762, in_queue=50784, util=84.86% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=512 --size=32255318425 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/9,526K /s] [0/19K iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28413 write: io=547MB, bw=9,343KB/s, iops=18,686, runt= 60001msec clat (usec): min=38, max=4,658, avg=50.41, stdev=39.33 bw (KB/s) : min= 4401, max= 9732, per=100.02%, avg=9344.61, stdev=551.20 cpu : usr=5.92%, sys=13.98%, ctx=1121617, majf=0, minf=3115 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1121195, short=0/0 lat (usec): 50=62.55%, 100=36.98%, 250=0.40%, 500=0.01%, 750=0.01% lat (usec): 1000=0.05% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=547MB, aggrb=9,343KB/s, minb=9,567KB/s, maxb=9,567KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/1118378, merge=0/0, ticks=0/44728, in_queue=44744, util=74.83% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/501M /s] [0/489 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28418 write: io=5,120MB, bw=478MB/s, iops=477, runt= 10719msec clat (usec): min=1,393, max=6,790, avg=2090.37, stdev=700.88 bw (KB/s) : min=219136, max=512000, per=100.03%, avg=489250.57, stdev=63031.26 cpu : usr=0.23%, sys=4.06%, ctx=5142, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=39.02%, 4=59.32%, 10=1.66% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=478MB/s, minb=489MB/s, maxb=489MB/s, mint=10719msec, maxt=10719msec Disk stats (read/write): fioa: ios=0/40852, merge=0/0, ticks=0/47575, in_queue=47585, util=97.33% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/502M /s] [0/490 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28423 write: io=5,120MB, bw=493MB/s, iops=493, runt= 10377msec clat (usec): min=1,386, max=4,042, avg=2023.46, stdev=530.92 bw (KB/s) : min=500758, max=512000, per=100.12%, avg=505865.40, stdev=2871.58 cpu : usr=0.31%, sys=4.07%, ctx=5146, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=39.75%, 4=60.23%, 10=0.02% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=493MB/s, minb=505MB/s, maxb=505MB/s, mint=10377msec, maxt=10377msec Disk stats (read/write): fioa: ios=0/40228, merge=0/0, ticks=0/44590, in_queue=44595, util=97.29% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [W] [100.0% done] [0K/502M /s] [0/490 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28426 write: io=29,208MB, bw=487MB/s, iops=486, runt= 60001msec clat (usec): min=1,390, max=6,719, avg=2050.84, stdev=601.24 bw (KB/s) : min=257019, max=509952, per=100.08%, avg=498873.25, stdev=32324.12 cpu : usr=0.41%, sys=3.88%, ctx=29327, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/29208, short=0/0 lat (msec): 2=39.64%, 4=59.77%, 10=0.59% Run status group 0 (all jobs): WRITE: io=29,208MB, aggrb=487MB/s, minb=498MB/s, maxb=498MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/233298, merge=0/0, ticks=0/262128, in_queue=262148, util=97.61% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/506M /s] [0/494 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28433 write: io=5,120MB, bw=492MB/s, iops=492, runt= 10406msec clat (usec): min=1,389, max=4,137, avg=2027.47, stdev=537.26 bw (KB/s) : min=496670, max=509952, per=100.17%, avg=504691.50, stdev=3252.76 cpu : usr=0.35%, sys=4.07%, ctx=5149, majf=0, minf=20 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=39.94%, 4=60.04%, 10=0.02% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=492MB/s, minb=504MB/s, maxb=504MB/s, mint=10406msec, maxt=10406msec Disk stats (read/write): fioa: ios=0/40139, merge=0/0, ticks=0/43966, in_queue=43971, util=97.16% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/506M /s] [0/494 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28436 write: io=5,120MB, bw=481MB/s, iops=480, runt= 10646msec clat (usec): min=1,384, max=7,578, avg=2074.72, stdev=683.05 bw (KB/s) : min=315392, max=508934, per=100.82%, avg=496500.19, stdev=41563.81 cpu : usr=0.21%, sys=3.64%, ctx=5134, majf=0, minf=20 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=39.34%, 4=59.26%, 10=1.41% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=481MB/s, minb=492MB/s, maxb=492MB/s, mint=10646msec, maxt=10646msec Disk stats (read/write): fioa: ios=0/40647, merge=0/0, ticks=0/46128, in_queue=46139, util=97.48% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=322553184256 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process Jobs: 1 (f=1): [w] [100.0% done] [0K/435M /s] [0/424 iops] [eta 00m:00s] file1: (groupid=0, jobs=1): err= 0: pid=28441 write: io=28,142MB, bw=469MB/s, iops=469, runt= 60001msec clat (usec): min=1,388, max=23,573, avg=2127.01, stdev=801.19 bw (KB/s) : min=258048, max=509952, per=100.07%, avg=480630.72, stdev=38998.43 cpu : usr=0.64%, sys=3.74%, ctx=28259, majf=0, minf=75 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/28142, short=0/0 lat (msec): 2=38.74%, 4=59.01%, 10=2.23%, 20=0.01%, 50=0.02% Run status group 0 (all jobs): WRITE: io=28,142MB, aggrb=469MB/s, minb=480MB/s, maxb=480MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/224786, merge=0/0, ticks=0/266407, in_queue=266427, util=97.32% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=512 --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/38M /s] [0/75K iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28447 write: io=2,103MB, bw=35,899KB/s, iops=71,797, runt= 60001msec clat (usec): min=24, max=5,131, avg=53.51, stdev=24.40 bw (KB/s) : min= 3894, max= 9937, per=24.98%, avg=8968.98, stdev=373.85 cpu : usr=5.41%, sys=13.12%, ctx=4310960, majf=0, minf=100 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/4307899, short=0/0 lat (usec): 50=36.29%, 100=63.29%, 250=0.33%, 500=0.01%, 750=0.01% lat (usec): 1000=0.06% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=2,103MB, aggrb=35,898KB/s, minb=36,760KB/s, maxb=36,760KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/4301177, merge=0/0, ticks=0/187190, in_queue=185980, util=95.74% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=512 --size=32255318425 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/38M /s] [0/76K iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28484 write: io=2,182MB, bw=37,247KB/s, iops=74,493, runt= 60001msec clat (usec): min=24, max=4,909, avg=51.56, stdev=23.45 bw (KB/s) : min= 4029, max=10199, per=25.10%, avg=9349.11, stdev=323.44 cpu : usr=5.63%, sys=13.39%, ctx=4472604, majf=0, minf=106 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/4469695, short=0/0 lat (usec): 50=49.76%, 100=49.88%, 250=0.28%, 500=0.01%, 750=0.01% lat (usec): 1000=0.05% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=2,182MB, aggrb=37,246KB/s, minb=38,140KB/s, maxb=38,140KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/4462286, merge=0/0, ticks=0/191016, in_queue=189919, util=97.03% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=512 --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 2 (f=2): [_ww_] [33.7% done] [0K/31M /s] [0/61K iops] [eta 02m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28492 write: io=1,799MB, bw=30,700KB/s, iops=61,400, runt= 60001msec clat (usec): min=25, max=13,864, avg=61.86, stdev=26.08 bw (KB/s) : min= 3371, max= 8727, per=24.89%, avg=7639.92, stdev=297.93 cpu : usr=5.73%, sys=13.40%, ctx=3721408, majf=0, minf=9058 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/3684088, short=0/0 lat (usec): 50=8.03%, 100=91.35%, 250=0.51%, 500=0.01%, 750=0.01% lat (usec): 1000=0.06% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01% Run status group 0 (all jobs): WRITE: io=1,799MB, aggrb=30,700KB/s, minb=31,437KB/s, maxb=31,437KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/3682781, merge=0/0, ticks=0/181535, in_queue=180331, util=91.29% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=512 --size=32255318425 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 2 (f=2): [__ww] [33.7% done] [0K/33M /s] [0/67K iops] [eta 02m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28500 write: io=1,879MB, bw=32,063KB/s, iops=64,126, runt= 60001msec clat (usec): min=25, max=5,145, avg=59.32, stdev=23.95 bw (KB/s) : min= 3421, max= 9003, per=24.92%, avg=7990.49, stdev=287.45 cpu : usr=6.08%, sys=12.84%, ctx=3877825, majf=0, minf=13579 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/3847633, short=0/0 lat (usec): 50=12.55%, 100=86.92%, 250=0.43%, 500=0.01%, 750=0.01% lat (usec): 1000=0.06% lat (msec): 2=0.01%, 4=0.01%, 10=0.01% Run status group 0 (all jobs): WRITE: io=1,879MB, aggrb=32,063KB/s, minb=32,832KB/s, maxb=32,832KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/3845188, merge=0/0, ticks=0/179510, in_queue=178908, util=92.95% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/337M /s] [0/329 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28510 write: io=20,480MB, bw=353MB/s, iops=352, runt= 58037msec clat (msec): min=1, max=443, avg=11.33, stdev= 9.37 bw (KB/s) : min=18177, max=132063, per=25.29%, avg=91369.07, stdev=12488.53 cpu : usr=0.06%, sys=0.83%, ctx=20584, majf=0, minf=100 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.01%, 10=96.33%, 20=0.08%, 100=2.32%, 250=1.22% lat (msec): 500=0.04% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=353MB/s, minb=361MB/s, maxb=361MB/s, mint=58037msec, maxt=58037msec Disk stats (read/write): fioa: ios=0/163556, merge=0/0, ticks=0/1696057, in_queue=1696152, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/255M /s] [0/249 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28518 write: io=16,150MB, bw=269MB/s, iops=269, runt= 60037msec clat (msec): min=2, max=476, avg=14.86, stdev=15.06 bw (KB/s) : min=17130, max=132063, per=25.31%, avg=69717.79, stdev=11571.74 cpu : usr=0.03%, sys=0.58%, ctx=16260, majf=0, minf=108 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/16150, short=0/0 lat (msec): 4=0.01%, 10=94.49%, 20=0.10%, 100=0.66%, 250=4.69% lat (msec): 500=0.05% Run status group 0 (all jobs): WRITE: io=16,150MB, aggrb=269MB/s, minb=275MB/s, maxb=275MB/s, mint=60037msec, maxt=60037msec Disk stats (read/write): fioa: ios=0/129168, merge=0/0, ticks=0/1796129, in_queue=1796360, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [WWWW] [100.0% done] [0K/251M /s] [0/245 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28526 write: io=20,590MB, bw=343MB/s, iops=343, runt= 60008msec clat (msec): min=2, max=407, avg=11.65, stdev= 8.88 bw (KB/s) : min=26570, max=130810, per=25.14%, avg=88323.32, stdev=8664.36 cpu : usr=0.07%, sys=0.91%, ctx=20728, majf=0, minf=100 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20590, short=0/0 lat (msec): 4=0.01%, 10=95.16%, 20=0.17%, 50=0.06%, 100=4.16% lat (msec): 250=0.41%, 500=0.04% Run status group 0 (all jobs): WRITE: io=20,590MB, aggrb=343MB/s, minb=351MB/s, maxb=351MB/s, mint=60008msec, maxt=60008msec Disk stats (read/write): fioa: ios=0/164324, merge=0/0, ticks=0/1757307, in_queue=1757391, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [100.0% done] [0K/408M /s] [0/398 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28534 write: io=20,480MB, bw=416MB/s, iops=415, runt= 49256msec clat (msec): min=1, max=330, avg= 9.61, stdev= 4.49 bw (KB/s) : min=62577, max=131072, per=25.10%, avg=106878.86, stdev=5346.99 cpu : usr=0.07%, sys=0.87%, ctx=20570, majf=0, minf=104 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=95.73%, 20=0.08%, 50=3.23% lat (msec): 100=0.91%, 250=0.02%, 500=0.02% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=416MB/s, minb=426MB/s, maxb=426MB/s, mint=49256msec, maxt=49256msec Disk stats (read/write): fioa: ios=0/163403, merge=0/0, ticks=0/1415394, in_queue=1415493, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [100.0% done] [0K/443M /s] [0/432 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28542 write: io=20,480MB, bw=446MB/s, iops=446, runt= 45878msec clat (msec): min=2, max=378, avg= 8.96, stdev= 4.24 bw (KB/s) : min=36644, max=130549, per=25.07%, avg=114591.94, stdev=6520.47 cpu : usr=0.07%, sys=1.11%, ctx=20617, majf=0, minf=104 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 4=0.01%, 10=96.30%, 20=0.21%, 50=3.44%, 500=0.04% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=446MB/s, minb=457MB/s, maxb=457MB/s, mint=45878msec, maxt=45878msec Disk stats (read/write): fioa: ios=0/163700, merge=0/0, ticks=0/1310307, in_queue=1310391, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=322553184256 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes Jobs: 4 (f=4): [wwww] [100.0% done] [0K/215M /s] [0/210 iops] [eta 00m:00s] file1: (groupid=0, jobs=4): err= 0: pid=28579 write: io=18,167MB, bw=303MB/s, iops=302, runt= 60009msec clat (msec): min=2, max=490, avg=13.21, stdev=13.39 bw (KB/s) : min= 6059, max=131072, per=25.24%, avg=78258.55, stdev=14811.26 cpu : usr=0.05%, sys=0.75%, ctx=18304, majf=0, minf=170 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/18167, short=0/0 lat (msec): 4=0.01%, 10=95.31%, 20=0.56%, 50=0.04%, 100=0.57% lat (msec): 250=3.46%, 500=0.04% Run status group 0 (all jobs): WRITE: io=18,167MB, aggrb=303MB/s, minb=310MB/s, maxb=310MB/s, mint=60009msec, maxt=60009msec Disk stats (read/write): fioa: ios=0/145064, merge=0/0, ticks=0/1770031, in_queue=1773707, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=512 --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/92M /s] [0/183K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28588 write: io=5,787MB, bw=98,763KB/s, iops=198K, runt= 60001msec clat (usec): min=26, max=24,318, avg=323.27, stdev=44.77 bw (KB/s) : min= 525, max= 1933, per=1.57%, avg=1546.81, stdev=27.69 cpu : usr=0.67%, sys=3.36%, ctx=11919458, majf=0, minf=1909 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/11851815, short=0/0 lat (usec): 50=0.02%, 100=0.23%, 250=15.60%, 500=82.52%, 750=0.96% lat (usec): 1000=0.11% lat (msec): 2=0.42%, 4=0.10%, 10=0.02%, 20=0.01%, 50=0.02% Run status group 0 (all jobs): WRITE: io=5,787MB, aggrb=98,763KB/s, minb=99MB/s, maxb=99MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/11825847, merge=0/0, ticks=0/3598999, in_queue=3567083, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=512 --size=32255318425 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/88M /s] [0/177K iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28658 write: io=5,496MB, bw=93,790KB/s, iops=188K, runt= 60002msec clat (usec): min=28, max=23,877, avg=340.86, stdev=35.48 bw (KB/s) : min= 888, max= 1913, per=1.56%, avg=1461.99, stdev=21.97 cpu : usr=0.57%, sys=2.69%, ctx=11314107, majf=0, minf=1717 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/11255195, short=0/0 lat (usec): 50=0.01%, 100=0.24%, 250=8.78%, 500=89.17%, 750=1.15% lat (usec): 1000=0.11% lat (msec): 2=0.42%, 4=0.08%, 10=0.01%, 50=0.01% Run status group 0 (all jobs): WRITE: io=5,496MB, aggrb=93,790KB/s, minb=96,041KB/s, maxb=96,041KB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): fioa: ios=0/11233772, merge=0/0, ticks=0/3619338, in_queue=3594081, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=512 --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 13 (f=13): [___________________w_______wwwwwww______w____________wwww_______] [2.0% done] [0K/62M /s] [0/123K iops] [eta 51m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28726 write: io=4,004MB, bw=68,337KB/s, iops=137K, runt= 60001msec clat (usec): min=31, max=7,197, avg=471.64, stdev=24.04 bw (KB/s) : min= 384, max= 6982, per=1.54%, avg=1051.23, stdev=31.98 cpu : usr=0.74%, sys=2.55%, ctx=8356881, majf=0, minf=57044 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/8200621, short=0/0 lat (usec): 50=0.12%, 100=1.82%, 250=6.31%, 500=56.91%, 750=31.01% lat (usec): 1000=2.78% lat (msec): 2=0.91%, 4=0.11%, 10=0.03% Run status group 0 (all jobs): WRITE: io=4,004MB, aggrb=68,337KB/s, minb=69,977KB/s, maxb=69,977KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): fioa: ios=0/8192460, merge=0/0, ticks=0/3649254, in_queue=3626466, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=512 --size=32255318425 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 12 (f=12): [_______________wwwwwwwwww_w_w___________________________________] [2.0% done] [0K/68M /s] [0/136K iops] [eta 52m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28794 write: io=3,759MB, bw=64,157KB/s, iops=128K, runt= 60003msec clat (usec): min=30, max=11,952, avg=499.80, stdev=57.49 bw (KB/s) : min= 134, max= 6893, per=1.55%, avg=992.87, stdev=53.84 cpu : usr=0.67%, sys=2.26%, ctx=7819499, majf=0, minf=54163 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/7699232, short=0/0 lat (usec): 50=0.18%, 100=2.11%, 250=8.25%, 500=56.78%, 750=28.28% lat (usec): 1000=2.06% lat (msec): 2=0.88%, 4=0.95%, 10=0.50%, 20=0.01% Run status group 0 (all jobs): WRITE: io=3,759MB, aggrb=64,157KB/s, minb=65,696KB/s, maxb=65,696KB/s, mint=60003msec, maxt=60003msec Disk stats (read/write): fioa: ios=0/7692746, merge=0/0, ticks=0/3678427, in_queue=3660378, util=99.94% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 25 (f=25): [WWE___W__WW_____W__E_WWW_E_W___W_WE____W_WWWW_W_W__EW_W_W_W_W__W] [7.3% done] [0K/330M /s] [0/322 iops] [eta 12m:58s] file1: (groupid=0, jobs=64): err= 0: pid=28865 write: io=24,221MB, bw=402MB/s, iops=402, runt= 60198msec clat (msec): min=1, max=500, avg=158.38, stdev= 5.06 bw (KB/s) : min= 3047, max=12709, per=1.58%, avg=6495.78, stdev=114.68 cpu : usr=0.00%, sys=0.06%, ctx=48030, majf=0, minf=1845 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/24221, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.02%, 50=2.29% lat (msec): 100=0.48%, 250=96.40%, 500=0.78%, 750=0.01% Run status group 0 (all jobs): WRITE: io=24,221MB, aggrb=402MB/s, minb=412MB/s, maxb=412MB/s, mint=60198msec, maxt=60198msec Disk stats (read/write): fioa: ios=0/193574, merge=0/0, ticks=0/7898688, in_queue=7900760, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 64 (f=64): [WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW] [100.0% done] [0K/388M /s] [0/378 iops] [eta 00m:00s] file1: (groupid=0, jobs=64): err= 0: pid=28935 write: io=25,666MB, bw=427MB/s, iops=426, runt= 60126msec clat (msec): min=1, max=438, avg=149.56, stdev= 3.78 bw (KB/s) : min= 3390, max=11420, per=1.57%, avg=6869.76, stdev=100.26 cpu : usr=0.00%, sys=0.06%, ctx=51144, majf=0, minf=1589 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/25666, short=0/0 lat (msec): 2=0.01%, 10=0.01%, 20=0.03%, 50=1.70%, 100=0.43% lat (msec): 250=97.33%, 500=0.51% Run status group 0 (all jobs): WRITE: io=25,666MB, aggrb=427MB/s, minb=437MB/s, maxb=437MB/s, mint=60126msec, maxt=60126msec Disk stats (read/write): fioa: ios=0/204610, merge=0/0, ticks=0/7866783, in_queue=7868841, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=write --bs=1m --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 3 (f=2): [______W_______________________E________E_______W_______W_E___E__] [1.8% done] [0K/401M /s] [0/391 iops] [eta 56m:59s] s] file1: (groupid=0, jobs=64): err= 0: pid=29003 write: io=26,062MB, bw=433MB/s, iops=433, runt= 60153msec clat (msec): min=1, max=454, avg=147.13, stdev= 3.85 bw (KB/s) : min= 3442, max=11658, per=1.58%, avg=6988.54, stdev=98.07 cpu : usr=0.00%, sys=0.07%, ctx=51846, majf=0, minf=1909 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/26062, short=0/0 lat (msec): 2=0.01%, 10=0.02%, 20=0.02%, 50=1.88%, 100=0.53% lat (msec): 250=97.05%, 500=0.49% Run status group 0 (all jobs): WRITE: io=26,062MB, aggrb=433MB/s, minb=444MB/s, maxb=444MB/s, mint=60153msec, maxt=60153msec Disk stats (read/write): fioa: ios=0/208480, merge=0/0, ticks=0/7903328, in_queue=7903746, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 37 (f=37): [wwwwwwwEEw_w_www_EE__www__www__wwwwwEww____w_www___w_w__w_w_wwww] [5.6% done] [0K/281M /s] [0/274 iops] [eta 17m:18s] file1: (groupid=0, jobs=64): err= 0: pid=29071 write: io=18,501MB, bw=307MB/s, iops=306, runt= 60341msec clat (msec): min=1, max=557, avg=209.77, stdev= 7.24 bw (KB/s) : min= 1851, max=11335, per=1.60%, avg=5033.99, stdev=172.01 cpu : usr=0.00%, sys=0.05%, ctx=36891, majf=0, minf=1459 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/18501, short=0/0 lat (msec): 2=0.01%, 10=0.02%, 20=0.03%, 50=1.57%, 100=0.14% lat (msec): 250=93.61%, 500=3.94%, 750=0.69% Run status group 0 (all jobs): WRITE: io=18,501MB, aggrb=307MB/s, minb=314MB/s, maxb=314MB/s, mint=60341msec, maxt=60341msec Disk stats (read/write): fioa: ios=0/147715, merge=0/0, ticks=0/7867468, in_queue=7869582, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=317184475136 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 8 (f=8): [_______w____________________w_w_____ww__________w___________w_w_] [4.8% done] [0K/229M /s] [0/223 iops] [eta 20m:08s] s] file1: (groupid=0, jobs=64): err= 0: pid=29139 write: io=16,023MB, bw=266MB/s, iops=265, runt= 60267msec clat (msec): min=2, max=707, avg=243.01, stdev= 8.69 bw (KB/s) : min= 1448, max=11468, per=1.57%, avg=4274.97, stdev=161.42 cpu : usr=0.00%, sys=0.04%, ctx=31849, majf=0, minf=2007 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/16023, short=0/0 lat (msec): 4=0.01%, 10=0.02%, 20=0.04%, 50=1.80%, 100=0.16% lat (msec): 250=48.43%, 500=49.05%, 750=0.48% Run status group 0 (all jobs): WRITE: io=16,023MB, aggrb=266MB/s, minb=272MB/s, maxb=272MB/s, mint=60267msec, maxt=60267msec Disk stats (read/write): fioa: ios=0/128120, merge=0/0, ticks=0/7851113, in_queue=7854669, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: /usr/bin/fio --filename=/dev/fioa --direct=1 --rw=randwrite --bs=1m --size=322553184256 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes Jobs: 26 (f=26): [w__w__w_ww__w___w_ww___ww_w__w_______w___www____ww_www___w__w_ww] [2.6% done] [0K/156M /s] [0/152 iops] [eta 37m:59s] file1: (groupid=0, jobs=64): err= 0: pid=29236 write: io=10,302MB, bw=171MB/s, iops=170, runt= 60341msec clat (msec): min=2, max=837, avg=377.02, stdev=20.25 bw (KB/s) : min= 1223, max=11548, per=1.62%, avg=2830.76, stdev=208.52 cpu : usr=0.00%, sys=0.03%, ctx=20514, majf=0, minf=1459 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/10302, short=0/0 lat (msec): 4=0.01%, 10=0.04%, 20=0.05%, 50=1.54%, 100=0.26% lat (msec): 250=20.82%, 500=36.23%, 750=40.03%, 1000=1.02% Run status group 0 (all jobs): WRITE: io=10,302MB, aggrb=171MB/s, minb=175MB/s, maxb=175MB/s, mint=60341msec, maxt=60341msec Disk stats (read/write): fioa: ios=0/82209, merge=0/0, ticks=0/7812430, in_queue=7822385, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ [root@iotest ~]#