################################################################################# Executing read tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8699 read : io=345MB, bw=5,887KB/s, iops=11,774, runt= 60001msec clat (usec): min=64, max=139K, avg=80.41, stdev=299.66 bw (KB/s) : min= 2450, max= 6177, per=100.11%, avg=5893.54, stdev=406.35 cpu : usr=11.12%, sys=21.98%, ctx=706904, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=706459/0, short=0/0 lat (usec): 100=91.37%, 250=8.60%, 500=0.03% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 50=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=345MB, aggrb=5,887KB/s, minb=6,028KB/s, maxb=6,028KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=705115/0, merge=0/0, ticks=47890/0, in_queue=47986, util=80.15% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8712 read : io=344MB, bw=5,872KB/s, iops=11,743, runt= 60001msec clat (usec): min=65, max=135K, avg=80.69, stdev=295.35 bw (KB/s) : min= 3138, max= 6132, per=100.05%, avg=5874.03, stdev=350.76 cpu : usr=12.39%, sys=25.43%, ctx=705171, majf=0, minf=28 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=704622/0, short=0/0 lat (usec): 100=91.40%, 250=8.59%, 500=0.01%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=344MB, aggrb=5,871KB/s, minb=6,012KB/s, maxb=6,012KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=703272/0, merge=0/0, ticks=47403/0, in_queue=47487, util=79.32% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8718 read : io=159MB, bw=2,706KB/s, iops=5,411, runt= 60001msec clat (usec): min=72, max=135K, avg=178.75, stdev=476.42 bw (KB/s) : min= 935, max= 2820, per=100.10%, avg=2707.61, stdev=219.73 cpu : usr=4.87%, sys=15.72%, ctx=324972, majf=0, minf=3065 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=324685/0, short=0/0 lat (usec): 100=0.01%, 250=97.90%, 500=2.06%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=159MB, aggrb=2,705KB/s, minb=2,770KB/s, maxb=2,770KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=323765/0, merge=0/0, ticks=52786/0, in_queue=52775, util=88.15% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8752 read : io=160MB, bw=2,738KB/s, iops=5,475, runt= 60001msec clat (usec): min=73, max=135K, avg=176.57, stdev=443.65 bw (KB/s) : min= 1214, max= 2846, per=100.11%, avg=2739.92, stdev=203.61 cpu : usr=5.79%, sys=15.11%, ctx=328960, majf=0, minf=3118 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=328545/0, short=0/0 lat (usec): 100=0.01%, 250=98.54%, 500=1.42%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=160MB, aggrb=2,737KB/s, minb=2,803KB/s, maxb=2,803KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=327561/0, merge=0/0, ticks=51460/0, in_queue=51336, util=85.75% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8758 read : io=5,120MB, bw=725MB/s, iops=725, runt= 7062msec clat (usec): min=1,302, max=9,140, avg=1374.18, stdev=239.87 bw (KB/s) : min=729088, max=750115, per=100.09%, avg=743038.36, stdev=7833.32 cpu : usr=0.27%, sys=8.16%, ctx=5146, majf=0, minf=280 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.77%, 4=0.06%, 10=0.18% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=725MB/s, minb=742MB/s, maxb=742MB/s, mint=7062msec, maxt=7062msec Disk stats (read/write): sdb: ios=10042/0, merge=0/0, ticks=10980/0, in_queue=10980, util=93.24% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8761 read : io=5,120MB, bw=662MB/s, iops=662, runt= 7733msec clat (usec): min=1,389, max=9,975, avg=1505.61, stdev=252.39 bw (KB/s) : min=663552, max=684032, per=100.32%, avg=680164.47, stdev=6814.24 cpu : usr=0.00%, sys=0.67%, ctx=5122, majf=0, minf=280 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.82%, 4=0.02%, 10=0.16% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=662MB/s, minb=678MB/s, maxb=678MB/s, mint=7733msec, maxt=7733msec Disk stats (read/write): sdb: ios=10169/0, merge=0/0, ticks=12755/0, in_queue=12765, util=98.20% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8766 read : io=42,133MB, bw=702MB/s, iops=702, runt= 60001msec clat (msec): min=1, max=139, avg= 1.42, stdev= 1.30 bw (KB/s) : min=201112, max=737280, per=100.21%, avg=720574.80, stdev=57188.74 cpu : usr=0.41%, sys=9.00%, ctx=42388, majf=0, minf=281 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=42133/0, short=0/0 lat (msec): 2=99.78%, 4=0.04%, 10=0.16%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=42,133MB, aggrb=702MB/s, minb=719MB/s, maxb=719MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=84102/0, merge=0/0, ticks=94905/0, in_queue=94905, util=94.47% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8771 read : io=5,120MB, bw=668MB/s, iops=668, runt= 7662msec clat (usec): min=1,353, max=8,918, avg=1489.91, stdev=233.63 bw (KB/s) : min=662227, max=694272, per=100.04%, avg=684534.07, stdev=9446.21 cpu : usr=0.47%, sys=5.81%, ctx=5138, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.80%, 4=0.06%, 10=0.14% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=668MB/s, minb=684MB/s, maxb=684MB/s, mint=7662msec, maxt=7662msec Disk stats (read/write): sdb: ios=9918/0, merge=0/0, ticks=12039/0, in_queue=12041, util=94.08% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8775 read : io=5,120MB, bw=634MB/s, iops=633, runt= 8077msec clat (usec): min=1,421, max=9,900, avg=1570.76, stdev=270.51 bw (KB/s) : min=633612, max=661504, per=100.06%, avg=649488.25, stdev=8002.88 cpu : usr=0.45%, sys=7.21%, ctx=5150, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.77%, 4=0.02%, 10=0.21% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=634MB/s, minb=649MB/s, maxb=649MB/s, mint=8077msec, maxt=8077msec Disk stats (read/write): sdb: ios=10048/0, merge=0/0, ticks=13253/0, in_queue=13253, util=95.02% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=8779 read : io=38,056MB, bw=634MB/s, iops=634, runt= 60001msec clat (msec): min=1, max=135, avg= 1.57, stdev= 1.35 bw (KB/s) : min=95812, max=702464, per=100.05%, avg=649791.64, stdev=58935.02 cpu : usr=0.40%, sys=6.59%, ctx=38226, majf=0, minf=312 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=38056/0, short=0/0 lat (msec): 2=99.76%, 4=0.02%, 10=0.18%, 20=0.02%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=38,056MB, aggrb=634MB/s, minb=649MB/s, maxb=649MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=75956/0, merge=0/0, ticks=99598/0, in_queue=99605, util=95.77% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8786 read : io=302MB, bw=5,150KB/s, iops=10,300, runt= 60001msec clat (usec): min=65, max=271K, avg=33647.64, stdev=2562.60 bw (KB/s) : min= 7, max= 5564, per=4.39%, avg=226.00, stdev=34.40 cpu : usr=2.85%, sys=6.43%, ctx=618618, majf=0, minf=116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=618066/0, short=0/0 lat (usec): 100=90.74%, 250=8.74%, 500=0.01%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.20%, 20=0.06%, 50=0.01% lat (msec): 100=0.01%, 250=0.25%, 500=0.01% Run status group 0 (all jobs): READ: io=302MB, aggrb=5,150KB/s, minb=5,274KB/s, maxb=5,274KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=615182/0, merge=1722/0, ticks=109024/0, in_queue=109172, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8795 read : io=201MB, bw=3,427KB/s, iops=6,854, runt= 60007msec clat (usec): min=65, max=419K, avg=3869.47, stdev=2881.71 bw (KB/s) : min= 4, max= 5236, per=38.92%, avg=1333.94, stdev=625.89 cpu : usr=1.79%, sys=4.14%, ctx=411673, majf=0, minf=111 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=411314/0, short=0/0 lat (usec): 100=89.10%, 250=9.25%, 500=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=1.41%, 50=0.01% lat (msec): 100=0.01%, 250=0.19%, 500=0.01% Run status group 0 (all jobs): READ: io=201MB, aggrb=3,427KB/s, minb=3,509KB/s, maxb=3,509KB/s, mint=60007msec, maxt=60007msec Disk stats (read/write): sdb: ios=408083/0, merge=3169/0, ticks=148028/0, in_queue=148109, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8803 read : io=588MB, bw=10,043KB/s, iops=20,085, runt= 60001msec clat (usec): min=72, max=137K, avg=192.22, stdev=243.11 bw (KB/s) : min= 443, max= 2634, per=25.04%, avg=2514.28, stdev=95.41 cpu : usr=6.75%, sys=9.92%, ctx=1211013, majf=0, minf=8823 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1205144/0, short=0/0 lat (usec): 100=0.01%, 250=95.01%, 500=4.94%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=588MB, aggrb=10,042KB/s, minb=10,283KB/s, maxb=10,283KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=1204474/0, merge=0/0, ticks=203339/0, in_queue=203040, util=99.62% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8812 read : io=606MB, bw=10,346KB/s, iops=20,691, runt= 60001msec clat (usec): min=75, max=135K, avg=186.42, stdev=228.92 bw (KB/s) : min= 486, max= 2707, per=25.02%, avg=2587.81, stdev=109.20 cpu : usr=8.23%, sys=11.27%, ctx=1246863, majf=0, minf=12516 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=1241505/0, short=0/0 lat (usec): 100=0.01%, 250=96.73%, 500=3.22%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=606MB, aggrb=10,345KB/s, minb=10,593KB/s, maxb=10,593KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=1240657/0, merge=0/0, ticks=196053/0, in_queue=195746, util=99.61% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8820 read : io=20,480MB, bw=617MB/s, iops=617, runt= 33183msec clat (msec): min=1, max=311, avg= 6.40, stdev=18.45 bw (KB/s) : min=82092, max=290816, per=27.06%, avg=171037.59, stdev=36834.92 cpu : usr=0.08%, sys=1.35%, ctx=20571, majf=0, minf=1125 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=98.19%, 4=0.03%, 10=0.16%, 20=0.01%, 100=0.01% lat (msec): 250=0.21%, 500=1.39% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=617MB/s, minb=632MB/s, maxb=632MB/s, mint=33183msec, maxt=33183msec Disk stats (read/write): sdb: ios=40756/0, merge=0/0, ticks=185738/0, in_queue=185869, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8857 read : io=20,480MB, bw=521MB/s, iops=521, runt= 39306msec clat (msec): min=1, max=589, avg= 7.45, stdev=20.43 bw (KB/s) : min= 2238, max=318850, per=27.84%, avg=148518.07, stdev=33729.93 cpu : usr=0.13%, sys=1.57%, ctx=20609, majf=0, minf=1126 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=87.22%, 4=10.58%, 10=0.23%, 20=0.07%, 100=0.01% lat (msec): 250=0.14%, 500=1.74%, 750=0.01% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=521MB/s, minb=534MB/s, maxb=534MB/s, mint=39306msec, maxt=39306msec Disk stats (read/write): sdb: ios=40956/0, merge=0/0, ticks=227500/0, in_queue=227528, util=99.94% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8865 read : io=34,904MB, bw=582MB/s, iops=581, runt= 60005msec clat (msec): min=1, max=515, avg= 6.91, stdev=19.71 bw (KB/s) : min= 9280, max=249357, per=26.97%, avg=160657.10, stdev=36819.54 cpu : usr=0.11%, sys=1.45%, ctx=35076, majf=0, minf=1142 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=34904/0, short=0/0 lat (msec): 2=97.99%, 4=0.08%, 10=0.21%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01%, 500=1.68%, 750=0.01% Run status group 0 (all jobs): READ: io=34,904MB, aggrb=582MB/s, minb=596MB/s, maxb=596MB/s, mint=60005msec, maxt=60005msec Disk stats (read/write): sdb: ios=69683/0, merge=0/0, ticks=351168/0, in_queue=351984, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8873 read : io=20,480MB, bw=876MB/s, iops=876, runt= 23376msec clat (msec): min=1, max=183, avg= 4.48, stdev= 1.79 bw (KB/s) : min=45426, max=455792, per=25.46%, avg=228391.64, stdev=18183.91 cpu : usr=0.18%, sys=2.42%, ctx=20572, majf=0, minf=1117 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=4.59%, 4=11.39%, 10=83.42%, 20=0.37%, 50=0.10% lat (msec): 100=0.05%, 250=0.07% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=876MB/s, minb=897MB/s, maxb=897MB/s, mint=23376msec, maxt=23376msec Disk stats (read/write): sdb: ios=40786/0, merge=0/0, ticks=167426/0, in_queue=167588, util=99.79% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8886 read : io=20,480MB, bw=863MB/s, iops=863, runt= 23723msec clat (msec): min=1, max=199, avg= 4.52, stdev= 1.80 bw (KB/s) : min=90244, max=339968, per=25.28%, avg=223485.67, stdev=11379.15 cpu : usr=0.17%, sys=2.32%, ctx=20586, majf=0, minf=1117 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=3.78%, 4=11.81%, 10=83.78%, 20=0.40%, 50=0.09% lat (msec): 100=0.06%, 250=0.08% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=863MB/s, minb=884MB/s, maxb=884MB/s, mint=23723msec, maxt=23723msec Disk stats (read/write): sdb: ios=40707/0, merge=0/0, ticks=169287/0, in_queue=169348, util=99.79% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=8893 read : io=52,881MB, bw=881MB/s, iops=881, runt= 60002msec clat (msec): min=1, max=187, avg= 4.53, stdev= 1.35 bw (KB/s) : min=116039, max=233650, per=25.03%, avg=225909.22, stdev=7995.27 cpu : usr=0.15%, sys=2.26%, ctx=53116, majf=0, minf=1242 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=52881/0, short=0/0 lat (msec): 2=0.01%, 4=11.27%, 10=88.42%, 20=0.25%, 50=0.01% lat (msec): 100=0.02%, 250=0.02% Run status group 0 (all jobs): READ: io=52,881MB, aggrb=881MB/s, minb=902MB/s, maxb=902MB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdb: ios=105547/0, merge=0/0, ticks=441219/0, in_queue=441240, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8902 read : io=128MB, bw=2,182KB/s, iops=4,364, runt= 60015msec clat (usec): min=65, max=506K, avg=48576.05, stdev=4466.48 bw (KB/s) : min= 0, max= 3498, per=0.47%, avg=10.29, stdev= 1.05 cpu : usr=0.05%, sys=0.10%, ctx=264092, majf=0, minf=2069 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=261951/0, short=0/0 lat (usec): 100=70.69%, 250=9.87%, 500=0.25%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.87%, 20=1.45%, 50=8.14% lat (msec): 100=4.05%, 250=4.27%, 500=0.40%, 750=0.01% Run status group 0 (all jobs): READ: io=128MB, aggrb=2,182KB/s, minb=2,234KB/s, maxb=2,234KB/s, mint=60015msec, maxt=60015msec Disk stats (read/write): sdb: ios=217329/0, merge=44484/0, ticks=636156/0, in_queue=637644, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8971 read : io=134MB, bw=2,287KB/s, iops=4,574, runt= 60011msec clat (usec): min=65, max=501K, avg=46232.23, stdev=4064.57 bw (KB/s) : min= 0, max= 3733, per=1.53%, avg=34.89, stdev=21.11 cpu : usr=0.05%, sys=0.10%, ctx=277358, majf=0, minf=2001 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=274539/0, short=0/0 lat (usec): 100=70.62%, 250=10.20%, 500=0.07%, 750=0.01% lat (msec): 10=0.70%, 20=1.77%, 50=7.06%, 100=5.25%, 250=4.19% lat (msec): 500=0.13%, 750=0.01% Run status group 0 (all jobs): READ: io=134MB, aggrb=2,287KB/s, minb=2,342KB/s, maxb=2,342KB/s, mint=60011msec, maxt=60011msec Disk stats (read/write): sdb: ios=226467/0, merge=46832/0, ticks=543693/0, in_queue=543961, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9039 read : io=4,068MB, bw=69,431KB/s, iops=139K, runt= 60001msec clat (usec): min=49, max=202K, avg=455.53, stdev=91.32 bw (KB/s) : min= 172, max= 2797, per=1.55%, avg=1073.78, stdev=17.65 cpu : usr=2.43%, sys=3.48%, ctx=8393115, majf=0, minf=70449 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=8331822/0, short=0/0 lat (usec): 50=0.01%, 100=0.01%, 250=7.21%, 500=60.83%, 750=29.64% lat (usec): 1000=2.00% lat (msec): 2=0.22%, 4=0.02%, 10=0.06%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=4,068MB, aggrb=69,430KB/s, minb=71,097KB/s, maxb=71,097KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=8331733/0, merge=0/0, ticks=3540255/0, in_queue=3540151, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9107 read : io=3,732MB, bw=63,550KB/s, iops=127K, runt= 60129msec clat (usec): min=75, max=232K, avg=494.23, stdev=107.20 bw (KB/s) : min= 244, max= 2457, per=1.55%, avg=988.00, stdev=17.18 cpu : usr=2.24%, sys=3.12%, ctx=7678417, majf=0, minf=82156 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=7642340/0, short=0/0 lat (usec): 100=0.01%, 250=3.90%, 500=57.06%, 750=34.91%, 1000=2.44% lat (msec): 2=1.43%, 4=0.19%, 10=0.08%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=3,732MB, aggrb=63,549KB/s, minb=65,074KB/s, maxb=65,074KB/s, mint=60129msec, maxt=60129msec Disk stats (read/write): sdb: ios=7642270/0, merge=1/0, ticks=3562704/0, in_queue=3562803, util=99.86% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9205 read : io=39,550MB, bw=658MB/s, iops=658, runt= 60077msec clat (msec): min=1, max=6,640, avg=104.92, stdev=95.77 bw (KB/s) : min= 157, max=120832, per=1.36%, avg=9183.14, stdev=489.63 cpu : usr=0.01%, sys=0.11%, ctx=39884, majf=0, minf=18381 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=39550/0, short=0/0 lat (msec): 2=98.05%, 4=0.14%, 10=0.15%, 100=0.01%, 250=0.01% lat (msec): 500=0.01%, 750=0.01%, 1000=0.01%, 2000=0.05%, >=2000=1.55% Run status group 0 (all jobs): READ: io=39,550MB, aggrb=658MB/s, minb=674MB/s, maxb=674MB/s, mint=60077msec, maxt=60077msec Disk stats (read/write): sdb: ios=78836/0, merge=0/0, ticks=5367260/0, in_queue=5672238, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9274 read : io=33,241MB, bw=553MB/s, iops=553, runt= 60083msec clat (msec): min=1, max=6,674, avg=114.12, stdev=101.95 bw (KB/s) : min= 169, max=101823, per=1.52%, avg=8598.20, stdev=433.71 cpu : usr=0.01%, sys=0.10%, ctx=33466, majf=0, minf=18182 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=33241/0, short=0/0 lat (msec): 2=93.87%, 4=3.18%, 10=0.21%, 20=0.04%, 50=0.11% lat (msec): 100=0.45%, 250=0.02%, 500=0.11%, 750=0.11%, 1000=0.02% lat (msec): 2000=0.06%, >=2000=1.83% Run status group 0 (all jobs): READ: io=33,241MB, aggrb=553MB/s, minb=567MB/s, maxb=567MB/s, mint=60083msec, maxt=60083msec Disk stats (read/write): sdb: ios=66249/0, merge=0/0, ticks=5399189/0, in_queue=5704300, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9342 read : io=40,661MB, bw=677MB/s, iops=676, runt= 60076msec clat (msec): min=1, max=6,679, avg=92.76, stdev=93.45 bw (KB/s) : min= 157, max=116736, per=1.53%, avg=10606.74, stdev=456.38 cpu : usr=0.01%, sys=0.10%, ctx=40928, majf=0, minf=18445 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=40661/0, short=0/0 lat (msec): 2=98.20%, 4=0.02%, 10=0.16%, 100=0.01%, 250=0.01% lat (msec): 500=0.01%, 750=0.01%, 1000=0.01%, 2000=0.05%, >=2000=1.51% Run status group 0 (all jobs): READ: io=40,661MB, aggrb=677MB/s, minb=693MB/s, maxb=693MB/s, mint=60076msec, maxt=60076msec Disk stats (read/write): sdb: ios=81080/0, merge=0/0, ticks=5303429/0, in_queue=5609430, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9410 read : io=71,684MB, bw=1,194MB/s, iops=1,193, runt= 60052msec clat (msec): min=2, max=321, avg=53.15, stdev= 2.39 bw (KB/s) : min= 5120, max=26846, per=1.58%, avg=19281.13, stdev=280.85 cpu : usr=0.01%, sys=0.16%, ctx=72038, majf=0, minf=18168 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=71684/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=1.19%, 50=41.32%, 100=57.18% lat (msec): 250=0.16%, 500=0.14% Run status group 0 (all jobs): READ: io=71,684MB, aggrb=1,194MB/s, minb=1,222MB/s, maxb=1,222MB/s, mint=60052msec, maxt=60052msec Disk stats (read/write): sdb: ios=142926/0, merge=0/0, ticks=7616882/0, in_queue=7620204, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9479 read : io=69,913MB, bw=1,164MB/s, iops=1,164, runt= 60054msec clat (msec): min=1, max=326, avg=55.40, stdev= 2.26 bw (KB/s) : min= 6005, max=26450, per=1.56%, avg=18539.41, stdev=283.18 cpu : usr=0.01%, sys=0.15%, ctx=70234, majf=0, minf=18860 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=69913/0, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=1.01%, 50=38.72% lat (msec): 100=59.96%, 250=0.15%, 500=0.14% Run status group 0 (all jobs): READ: io=69,913MB, aggrb=1,164MB/s, minb=1,192MB/s, maxb=1,192MB/s, mint=60054msec, maxt=60054msec Disk stats (read/write): sdb: ios=139394/0, merge=0/0, ticks=7616348/0, in_queue=7620057, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9547 read : io=69,743MB, bw=1,161MB/s, iops=1,161, runt= 60055msec clat (msec): min=2, max=316, avg=56.11, stdev= 2.37 bw (KB/s) : min= 3650, max=26465, per=1.54%, avg=18297.94, stdev=295.53 cpu : usr=0.01%, sys=0.16%, ctx=70061, majf=0, minf=18712 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=69743/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=0.98%, 50=38.62%, 100=59.83% lat (msec): 250=0.45%, 500=0.11% Run status group 0 (all jobs): READ: io=69,743MB, aggrb=1,161MB/s, minb=1,189MB/s, maxb=1,189MB/s, mint=60055msec, maxt=60055msec Disk stats (read/write): sdb: ios=139037/0, merge=0/0, ticks=7613693/0, in_queue=7618158, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9617 write: io=453MB, bw=7,723KB/s, iops=15,446, runt= 60001msec clat (usec): min=40, max=138K, avg=60.17, stdev=262.64 bw (KB/s) : min= 4544, max= 8066, per=100.04%, avg=7726.03, stdev=465.01 cpu : usr=14.00%, sys=35.15%, ctx=927079, majf=0, minf=30 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/926830, short=0/0 lat (usec): 50=19.04%, 100=76.84%, 250=4.10%, 500=0.01%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): WRITE: io=453MB, aggrb=7,723KB/s, minb=7,908KB/s, maxb=7,908KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/925021, merge=0/0, ticks=0/44673, in_queue=44754, util=74.75% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9622 write: io=454MB, bw=7,747KB/s, iops=15,493, runt= 60001msec clat (usec): min=40, max=135K, avg=59.99, stdev=262.20 bw (KB/s) : min= 4224, max= 8102, per=100.03%, avg=7748.51, stdev=498.13 cpu : usr=13.90%, sys=36.31%, ctx=929834, majf=0, minf=24 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/929599, short=0/0 lat (usec): 50=17.66%, 100=78.64%, 250=3.70%, 500=0.01%, 750=0.01% lat (msec): 4=0.01%, 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=454MB, aggrb=7,746KB/s, minb=7,932KB/s, maxb=7,932KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/927808, merge=0/0, ticks=0/44970, in_queue=45062, util=75.26% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9656 write: io=153MB, bw=2,605KB/s, iops=5,210, runt= 60001msec clat (usec): min=71, max=135K, avg=185.69, stdev=497.92 bw (KB/s) : min= 1125, max= 2788, per=100.11%, avg=2607.92, stdev=239.10 cpu : usr=4.85%, sys=13.16%, ctx=312974, majf=0, minf=2837 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/312640, short=0/0 lat (usec): 100=0.05%, 250=95.92%, 500=3.35%, 750=0.58%, 1000=0.03% lat (msec): 2=0.02%, 4=0.01%, 10=0.03%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=153MB, aggrb=2,605KB/s, minb=2,667KB/s, maxb=2,667KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/311717, merge=0/0, ticks=0/53141, in_queue=53091, util=88.68% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9662 write: io=155MB, bw=2,652KB/s, iops=5,303, runt= 60001msec clat (usec): min=48, max=135K, avg=182.33, stdev=508.42 bw (KB/s) : min= 1384, max= 2812, per=100.08%, avg=2653.09, stdev=210.63 cpu : usr=5.26%, sys=12.03%, ctx=318571, majf=0, minf=1043 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/318243, short=0/0 lat (usec): 50=0.01%, 100=0.15%, 250=97.42%, 500=2.14%, 750=0.22% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.05%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=155MB, aggrb=2,651KB/s, minb=2,715KB/s, maxb=2,715KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/317296, merge=0/0, ticks=0/52693, in_queue=52644, util=87.93% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9667 write: io=5,120MB, bw=583MB/s, iops=582, runt= 8783msec clat (msec): min=1, max=35, avg= 1.71, stdev= 2.18 bw (KB/s) : min=463473, max=668359, per=100.31%, avg=598776.82, stdev=61706.45 cpu : usr=1.05%, sys=3.79%, ctx=5139, majf=0, minf=23 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=83.71%, 4=9.73%, 10=5.51%, 20=0.78%, 50=0.27% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=583MB/s, minb=597MB/s, maxb=597MB/s, mint=8783msec, maxt=8783msec Disk stats (read/write): sdb: ios=0/10132, merge=0/0, ticks=0/13824, in_queue=13825, util=96.03% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9671 write: io=5,120MB, bw=557MB/s, iops=557, runt= 9189msec clat (msec): min=1, max=20, avg= 1.79, stdev= 1.57 bw (KB/s) : min=509952, max=628736, per=100.11%, avg=571188.83, stdev=31993.52 cpu : usr=0.39%, sys=1.89%, ctx=5139, majf=0, minf=23 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=85.04%, 4=9.14%, 10=5.04%, 20=0.76%, 50=0.02% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=557MB/s, minb=571MB/s, maxb=571MB/s, mint=9189msec, maxt=9189msec Disk stats (read/write): sdb: ios=0/10238, merge=0/0, ticks=0/14965, in_queue=14967, util=97.54% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9675 write: io=31,849MB, bw=531MB/s, iops=530, runt= 60001msec clat (msec): min=1, max=134, avg= 1.88, stdev= 2.76 bw (KB/s) : min=246447, max=633612, per=100.12%, avg=544207.61, stdev=71413.99 cpu : usr=0.67%, sys=2.94%, ctx=31952, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/31849, short=0/0 lat (msec): 2=85.71%, 4=8.59%, 10=4.52%, 20=0.75%, 50=0.41% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=31,849MB, aggrb=531MB/s, minb=544MB/s, maxb=544MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/63552, merge=0/0, ticks=0/95766, in_queue=95780, util=97.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9681 write: io=5,120MB, bw=515MB/s, iops=514, runt= 9942msec clat (msec): min=1, max=17, avg= 1.93, stdev= 1.65 bw (KB/s) : min=491520, max=571392, per=99.65%, avg=525501.11, stdev=21262.15 cpu : usr=0.60%, sys=3.32%, ctx=5135, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=81.97%, 4=7.60%, 10=9.53%, 20=0.90% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=515MB/s, minb=527MB/s, maxb=527MB/s, mint=9942msec, maxt=9942msec Disk stats (read/write): sdb: ios=0/10232, merge=0/0, ticks=0/16147, in_queue=16150, util=96.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9685 write: io=5,120MB, bw=503MB/s, iops=502, runt= 10184msec clat (msec): min=1, max=22, avg= 1.98, stdev= 1.77 bw (KB/s) : min=448512, max=569344, per=99.89%, avg=514238.10, stdev=36548.66 cpu : usr=0.30%, sys=2.75%, ctx=5133, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=80.14%, 4=9.57%, 10=9.39%, 20=0.86%, 50=0.04% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=503MB/s, minb=515MB/s, maxb=515MB/s, mint=10184msec, maxt=10184msec Disk stats (read/write): sdb: ios=0/9966, merge=0/0, ticks=0/16338, in_queue=16339, util=97.53% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=9689 write: io=26,187MB, bw=436MB/s, iops=436, runt= 60001msec clat (msec): min=1, max=135, avg= 2.28, stdev= 2.90 bw (KB/s) : min=214183, max=513017, per=100.06%, avg=447198.55, stdev=41209.51 cpu : usr=0.48%, sys=2.77%, ctx=26279, majf=0, minf=77 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/26187, short=0/0 lat (msec): 2=80.85%, 4=6.40%, 10=11.49%, 20=1.16%, 50=0.08% lat (msec): 100=0.02%, 250=0.01% Run status group 0 (all jobs): WRITE: io=26,187MB, aggrb=436MB/s, minb=447MB/s, maxb=447MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/52258, merge=0/0, ticks=0/99599, in_queue=99624, util=98.19% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9695 write: io=921MB, bw=15,711KB/s, iops=31,421, runt= 60001msec clat (usec): min=37, max=139K, avg=122.37, stdev=170.20 bw (KB/s) : min= 2468, max= 5327, per=25.01%, avg=3929.14, stdev=249.89 cpu : usr=11.52%, sys=13.73%, ctx=1887500, majf=0, minf=111 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1885293, short=0/0 lat (usec): 50=0.61%, 100=38.24%, 250=60.68%, 500=0.46%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): WRITE: io=921MB, aggrb=15,710KB/s, minb=16,087KB/s, maxb=16,087KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/1882332, merge=0/0, ticks=0/184709, in_queue=184576, util=98.97% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9703 write: io=1,205MB, bw=20,559KB/s, iops=41,117, runt= 60001msec clat (usec): min=36, max=179K, avg=92.35, stdev=192.62 bw (KB/s) : min= 1729, max= 6944, per=25.02%, avg=5142.84, stdev=458.87 cpu : usr=17.11%, sys=17.42%, ctx=2470541, majf=0, minf=116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/2467062, short=0/0 lat (usec): 50=1.59%, 100=72.25%, 250=26.07%, 500=0.09%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): WRITE: io=1,205MB, aggrb=20,558KB/s, minb=21,051KB/s, maxb=21,051KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/2462372, merge=0/0, ticks=0/156229, in_queue=155986, util=96.53% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9741 write: io=484MB, bw=8,257KB/s, iops=16,513, runt= 60001msec clat (usec): min=58, max=135K, avg=235.71, stdev=384.45 bw (KB/s) : min= 633, max= 2374, per=25.01%, avg=2064.81, stdev=110.08 cpu : usr=6.25%, sys=8.99%, ctx=994792, majf=0, minf=7506 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/990839, short=0/0 lat (usec): 100=0.02%, 250=89.10%, 500=8.11%, 750=2.37%, 1000=0.14% lat (msec): 2=0.05%, 4=0.01%, 10=0.10%, 20=0.11%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=484MB, aggrb=8,256KB/s, minb=8,455KB/s, maxb=8,455KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/990191, merge=0/0, ticks=0/206282, in_queue=206027, util=99.63% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9750 write: io=430MB, bw=7,332KB/s, iops=14,663, runt= 60001msec clat (usec): min=51, max=135K, avg=266.09, stdev=346.33 bw (KB/s) : min= 756, max= 2108, per=25.02%, avg=1834.22, stdev=88.59 cpu : usr=6.19%, sys=8.88%, ctx=883558, majf=0, minf=13264 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/879836, short=0/0 lat (usec): 100=0.01%, 250=79.80%, 500=14.76%, 750=4.70%, 1000=0.37% lat (msec): 2=0.11%, 4=0.02%, 10=0.21%, 20=0.03%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=430MB, aggrb=7,331KB/s, minb=7,507KB/s, maxb=7,507KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/879318, merge=0/0, ticks=0/207468, in_queue=207252, util=99.73% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9758 write: io=20,480MB, bw=599MB/s, iops=598, runt= 34216msec clat (msec): min=1, max=28, avg= 6.68, stdev= 1.37 bw (KB/s) : min=91792, max=194276, per=25.07%, avg=153652.33, stdev=9420.39 cpu : usr=0.24%, sys=1.08%, ctx=20558, majf=0, minf=98 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.04%, 4=14.07%, 10=76.92%, 20=8.57%, 50=0.40% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=599MB/s, minb=613MB/s, maxb=613MB/s, mint=34216msec, maxt=34216msec Disk stats (read/write): sdb: ios=0/40744, merge=0/0, ticks=0/247356, in_queue=247388, util=99.91% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9766 write: io=20,480MB, bw=615MB/s, iops=615, runt= 33293msec clat (msec): min=1, max=29, avg= 6.50, stdev= 1.46 bw (KB/s) : min=122960, max=192127, per=25.06%, avg=157852.50, stdev=4635.43 cpu : usr=0.26%, sys=1.17%, ctx=20564, majf=0, minf=94 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.06%, 4=10.83%, 10=81.92%, 20=6.34%, 50=0.84% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=615MB/s, minb=630MB/s, maxb=630MB/s, mint=33293msec, maxt=33293msec Disk stats (read/write): sdb: ios=0/40937, merge=0/0, ticks=0/242286, in_queue=242286, util=99.87% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9773 write: io=35,039MB, bw=584MB/s, iops=583, runt= 60009msec clat (msec): min=2, max=183, avg= 6.85, stdev= 1.97 bw (KB/s) : min=41951, max=189328, per=25.06%, avg=149846.57, stdev=11568.85 cpu : usr=0.25%, sys=1.12%, ctx=35193, majf=0, minf=110 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/35039, short=0/0 lat (msec): 4=7.95%, 10=81.90%, 20=9.75%, 50=0.32%, 100=0.04% lat (msec): 250=0.03% Run status group 0 (all jobs): WRITE: io=35,039MB, aggrb=584MB/s, minb=598MB/s, maxb=598MB/s, mint=60009msec, maxt=60009msec Disk stats (read/write): sdb: ios=0/69971, merge=0/0, ticks=0/434902, in_queue=434934, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9784 write: io=20,480MB, bw=563MB/s, iops=562, runt= 36401msec clat (msec): min=1, max=185, avg= 7.08, stdev= 2.44 bw (KB/s) : min=61863, max=200704, per=25.09%, avg=144563.94, stdev=11983.83 cpu : usr=0.18%, sys=1.11%, ctx=20571, majf=0, minf=107 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.02%, 4=9.33%, 10=77.26%, 20=12.84%, 50=0.44% lat (msec): 100=0.05%, 250=0.06% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=563MB/s, minb=576MB/s, maxb=576MB/s, mint=36401msec, maxt=36401msec Disk stats (read/write): sdb: ios=0/40876, merge=0/0, ticks=0/264685, in_queue=264703, util=99.92% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9792 write: io=20,480MB, bw=583MB/s, iops=583, runt= 35114msec clat (msec): min=1, max=192, avg= 6.85, stdev= 2.64 bw (KB/s) : min=58379, max=199111, per=25.07%, avg=149730.23, stdev=10351.05 cpu : usr=0.20%, sys=1.14%, ctx=20570, majf=0, minf=98 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.01%, 4=12.92%, 10=76.87%, 20=9.56%, 50=0.48% lat (msec): 100=0.08%, 250=0.07% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=583MB/s, minb=597MB/s, maxb=597MB/s, mint=35114msec, maxt=35114msec Disk stats (read/write): sdb: ios=0/40878, merge=0/0, ticks=0/256833, in_queue=256865, util=99.92% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=9799 write: io=33,045MB, bw=551MB/s, iops=550, runt= 60002msec clat (msec): min=1, max=187, avg= 7.25, stdev= 3.03 bw (KB/s) : min=64105, max=193399, per=25.04%, avg=141190.30, stdev=9843.49 cpu : usr=0.21%, sys=1.07%, ctx=33203, majf=0, minf=132 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/33045, short=0/0 lat (msec): 2=0.01%, 4=22.53%, 10=62.48%, 20=13.83%, 50=0.86% lat (msec): 100=0.20%, 250=0.10% Run status group 0 (all jobs): WRITE: io=33,045MB, aggrb=551MB/s, minb=564MB/s, maxb=564MB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdb: ios=0/65943, merge=0/0, ticks=0/442639, in_queue=442650, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9809 write: io=5,633MB, bw=96,137KB/s, iops=192K, runt= 60001msec clat (usec): min=44, max=189K, avg=272.67, stdev=73.84 bw (KB/s) : min= 17, max= 2314, per=1.86%, avg=1786.94, stdev=18.97 cpu : usr=3.08%, sys=5.44%, ctx=11567461, majf=0, minf=2068 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/11536654, short=0/0 lat (usec): 50=0.01%, 100=0.26%, 250=40.48%, 500=53.56%, 750=3.51% lat (usec): 1000=0.84% lat (msec): 2=0.89%, 4=0.27%, 10=0.13%, 20=0.04%, 50=0.02% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=5,633MB, aggrb=96,137KB/s, minb=98,444KB/s, maxb=98,444KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/9896426, merge=0/1464338, ticks=0/2956903, in_queue=2955239, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9877 write: io=5,625MB, bw=95,992KB/s, iops=192K, runt= 60001msec clat (usec): min=44, max=207K, avg=508.31, stdev=97.36 bw (KB/s) : min= 112, max= 2265, per=1.01%, avg=969.86, stdev=49.08 cpu : usr=3.06%, sys=5.39%, ctx=11549636, majf=0, minf=1789 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/11519227, short=0/0 lat (usec): 50=0.01%, 100=0.28%, 250=39.03%, 500=54.50%, 750=3.77% lat (usec): 1000=0.93% lat (msec): 2=0.96%, 4=0.34%, 10=0.15%, 20=0.02%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=5,625MB, aggrb=95,991KB/s, minb=98,295KB/s, maxb=98,295KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/9875547, merge=0/1468780, ticks=0/2972944, in_queue=2971109, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=9945 write: io=1,681MB, bw=28,684KB/s, iops=57,368, runt= 60020msec clat (usec): min=56, max=200K, avg=1106.83, stdev=321.29 bw (KB/s) : min= 76, max= 2245, per=1.56%, avg=446.81, stdev=14.95 cpu : usr=1.07%, sys=1.70%, ctx=3529526, majf=0, minf=38261 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/3443275, short=0/0 lat (usec): 100=0.01%, 250=4.08%, 500=22.83%, 750=28.89%, 1000=22.04% lat (msec): 2=18.33%, 4=1.10%, 10=0.50%, 20=2.09%, 50=0.14% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=1,681MB, aggrb=28,684KB/s, minb=29,372KB/s, maxb=29,372KB/s, mint=60020msec, maxt=60020msec Disk stats (read/write): sdb: ios=0/3442307, merge=0/0, ticks=0/3654412, in_queue=3654395, util=99.90% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10043 write: io=1,402MB, bw=23,925KB/s, iops=47,850, runt= 60014msec clat (usec): min=61, max=198K, avg=1358.06, stdev=266.88 bw (KB/s) : min= 71, max= 1767, per=1.53%, avg=364.98, stdev= 6.48 cpu : usr=0.97%, sys=1.39%, ctx=2930463, majf=0, minf=75348 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/2871687, short=0/0 lat (usec): 100=0.01%, 250=2.03%, 500=11.04%, 750=17.19%, 1000=20.87% lat (msec): 2=41.29%, 4=4.39%, 10=1.58%, 20=1.54%, 50=0.06% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=1,402MB, aggrb=23,925KB/s, minb=24,499KB/s, maxb=24,499KB/s, mint=60014msec, maxt=60014msec Disk stats (read/write): sdb: ios=0/2871068, merge=0/6, ticks=0/3683418, in_queue=3683541, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10112 write: io=34,605MB, bw=576MB/s, iops=575, runt= 60116msec clat (msec): min=1, max=2,694, avg=111.42, stdev= 6.34 bw (KB/s) : min= 380, max=35378, per=1.57%, avg=9269.73, stdev=323.68 cpu : usr=0.02%, sys=0.04%, ctx=34741, majf=0, minf=1996 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/34605, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=9.77% lat (msec): 100=25.59%, 250=61.57%, 500=2.88%, 750=0.02%, 1000=0.02% lat (msec): 2000=0.05%, >=2000=0.03% Run status group 0 (all jobs): WRITE: io=34,605MB, aggrb=576MB/s, minb=589MB/s, maxb=589MB/s, mint=60116msec, maxt=60116msec Disk stats (read/write): sdb: ios=0/68932, merge=0/0, ticks=0/7552476, in_queue=7559085, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10180 write: io=35,729MB, bw=594MB/s, iops=594, runt= 60105msec clat (msec): min=1, max=2,925, avg=98.94, stdev= 4.51 bw (KB/s) : min= 350, max=35310, per=1.71%, avg=10405.40, stdev=448.19 cpu : usr=0.02%, sys=0.05%, ctx=35885, majf=0, minf=1681 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/35729, short=0/0 lat (msec): 2=0.01%, 10=0.01%, 20=0.06%, 50=4.72%, 100=31.45% lat (msec): 250=62.45%, 500=1.18%, 750=0.04%, 1000=0.01%, 2000=0.05% lat (msec): >=2000=0.04% Run status group 0 (all jobs): WRITE: io=35,729MB, aggrb=594MB/s, minb=609MB/s, maxb=609MB/s, mint=60105msec, maxt=60105msec Disk stats (read/write): sdb: ios=0/71164, merge=0/0, ticks=0/7581298, in_queue=7588423, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10248 write: io=34,936MB, bw=581MB/s, iops=580, runt= 60140msec clat (msec): min=1, max=2,734, avg=99.56, stdev= 7.92 bw (KB/s) : min= 374, max=39766, per=1.75%, avg=10406.84, stdev=587.45 cpu : usr=0.02%, sys=0.04%, ctx=35088, majf=0, minf=2130 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/34936, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.81%, 50=18.00% lat (msec): 100=18.97%, 250=56.87%, 500=5.18%, 750=0.03%, 1000=0.02% lat (msec): 2000=0.06%, >=2000=0.03% Run status group 0 (all jobs): WRITE: io=34,936MB, aggrb=581MB/s, minb=595MB/s, maxb=595MB/s, mint=60140msec, maxt=60140msec Disk stats (read/write): sdb: ios=0/69850, merge=0/0, ticks=0/7535566, in_queue=7538475, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10316 write: io=32,967MB, bw=547MB/s, iops=547, runt= 60249msec clat (msec): min=1, max=487, avg=116.60, stdev= 8.66 bw (KB/s) : min= 2809, max=19616, per=1.59%, avg=8887.85, stdev=303.82 cpu : usr=0.01%, sys=0.05%, ctx=33082, majf=0, minf=1591 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/32967, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.22%, 50=15.51% lat (msec): 100=32.21%, 250=46.57%, 500=5.46% Run status group 0 (all jobs): WRITE: io=32,967MB, aggrb=547MB/s, minb=560MB/s, maxb=560MB/s, mint=60249msec, maxt=60249msec Disk stats (read/write): sdb: ios=0/65923, merge=0/0, ticks=0/7590129, in_queue=7591735, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10385 write: io=34,069MB, bw=567MB/s, iops=566, runt= 60108msec clat (msec): min=1, max=480, avg=110.77, stdev= 8.22 bw (KB/s) : min= 3185, max=22173, per=1.60%, avg=9278.71, stdev=254.39 cpu : usr=0.01%, sys=0.05%, ctx=34204, majf=0, minf=2382 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/34069, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.26%, 50=16.41% lat (msec): 100=32.99%, 250=45.89%, 500=4.41% Run status group 0 (all jobs): WRITE: io=34,069MB, aggrb=567MB/s, minb=580MB/s, maxb=580MB/s, mint=60108msec, maxt=60108msec Disk stats (read/write): sdb: ios=0/67878, merge=0/0, ticks=0/7567464, in_queue=7578209, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=10454 write: io=31,040MB, bw=516MB/s, iops=516, runt= 60112msec clat (msec): min=1, max=537, avg=122.57, stdev= 8.63 bw (KB/s) : min= 1906, max=19504, per=1.60%, avg=8453.24, stdev=270.68 cpu : usr=0.01%, sys=0.04%, ctx=31159, majf=0, minf=1587 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/31040, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.19%, 50=11.73% lat (msec): 100=29.38%, 250=53.95%, 500=4.70%, 750=0.03% Run status group 0 (all jobs): WRITE: io=31,040MB, aggrb=516MB/s, minb=529MB/s, maxb=529MB/s, mint=60112msec, maxt=60112msec Disk stats (read/write): sdb: ios=0/61796, merge=0/0, ticks=0/7577035, in_queue=7585075, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------