################################################################################# Executing read tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6527 read : io=258MB, bw=4,406KB/s, iops=8,812, runt= 60001msec clat (usec): min=91, max=135K, avg=108.82, stdev=330.37 bw (KB/s) : min= 2532, max= 4655, per=100.03%, avg=4407.14, stdev=242.56 cpu : usr=5.04%, sys=14.34%, ctx=528969, majf=0, minf=25 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=528731/0, short=0/0 lat (usec): 100=24.73%, 250=75.15%, 500=0.11%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=258MB, aggrb=4,406KB/s, minb=4,511KB/s, maxb=4,511KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=527738/0, merge=0/0, ticks=53875/0, in_queue=53935, util=90.08% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6547 read : io=259MB, bw=4,425KB/s, iops=8,850, runt= 60001msec clat (usec): min=91, max=137K, avg=108.37, stdev=332.07 bw (KB/s) : min= 1890, max= 4638, per=100.11%, avg=4429.68, stdev=278.33 cpu : usr=5.47%, sys=15.79%, ctx=531358, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=531049/0, short=0/0 lat (usec): 100=31.27%, 250=68.66%, 500=0.06%, 750=0.01% lat (msec): 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=259MB, aggrb=4,425KB/s, minb=4,531KB/s, maxb=4,531KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=530036/0, merge=0/0, ticks=53535/0, in_queue=53602, util=89.53% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6577 read : io=158MB, bw=2,691KB/s, iops=5,381, runt= 60001msec clat (usec): min=70, max=135K, avg=179.78, stdev=451.92 bw (KB/s) : min= 1196, max= 2822, per=100.10%, avg=2692.73, stdev=208.12 cpu : usr=4.62%, sys=13.16%, ctx=323171, majf=0, minf=1738 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=322889/0, short=0/0 lat (usec): 100=0.03%, 250=97.31%, 500=2.62%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=158MB, aggrb=2,690KB/s, minb=2,755KB/s, maxb=2,755KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=321954/0, merge=0/0, ticks=53219/0, in_queue=53220, util=88.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6587 read : io=141MB, bw=2,404KB/s, iops=4,808, runt= 60001msec clat (usec): min=85, max=135K, avg=201.91, stdev=498.49 bw (KB/s) : min= 1165, max= 2497, per=100.04%, avg=2404.95, stdev=177.82 cpu : usr=2.68%, sys=7.82%, ctx=288701, majf=0, minf=2082 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=288509/0, short=0/0 lat (usec): 100=0.01%, 250=95.78%, 500=4.16%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=141MB, aggrb=2,404KB/s, minb=2,461KB/s, maxb=2,461KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=287719/0, merge=0/0, ticks=56010/0, in_queue=55992, util=93.53% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6612 read : io=5,120MB, bw=796MB/s, iops=795, runt= 6436msec clat (usec): min=1,100, max=8,553, avg=1251.91, stdev=149.99 bw (KB/s) : min=804864, max=821652, per=100.05%, avg=815003.08, stdev=4560.56 cpu : usr=0.50%, sys=7.61%, ctx=5147, majf=0, minf=281 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.88%, 4=0.04%, 10=0.08% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=796MB/s, minb=815MB/s, maxb=815MB/s, mint=6436msec, maxt=6436msec Disk stats (read/write): sdb: ios=10220/0, merge=0/0, ticks=10704/0, in_queue=10703, util=93.52% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6619 read : io=5,120MB, bw=612MB/s, iops=612, runt= 8365msec clat (msec): min=1, max=135, avg= 1.63, stdev= 3.66 bw (KB/s) : min=141312, max=684032, per=99.75%, avg=625203.13, stdev=155855.02 cpu : usr=0.00%, sys=0.45%, ctx=5127, majf=0, minf=280 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.73%, 4=0.02%, 10=0.10%, 50=0.02%, 100=0.08% lat (msec): 250=0.06% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=612MB/s, minb=627MB/s, maxb=627MB/s, mint=8365msec, maxt=8365msec Disk stats (read/write): sdb: ios=9994/0, merge=0/0, ticks=13867/0, in_queue=13869, util=98.58% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6624 read : io=37,707MB, bw=628MB/s, iops=628, runt= 60001msec clat (msec): min=1, max=135, avg= 1.59, stdev= 1.36 bw (KB/s) : min=263013, max=684032, per=100.15%, avg=644472.55, stdev=52446.44 cpu : usr=0.28%, sys=5.03%, ctx=37862, majf=0, minf=285 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=37707/0, short=0/0 lat (msec): 2=99.75%, 4=0.07%, 10=0.15%, 50=0.01%, 100=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): READ: io=37,707MB, aggrb=628MB/s, minb=644MB/s, maxb=644MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=75264/0, merge=0/0, ticks=98285/0, in_queue=98288, util=96.62% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6632 read : io=5,120MB, bw=649MB/s, iops=649, runt= 7885msec clat (usec): min=1,410, max=9,026, avg=1533.15, stdev=241.02 bw (KB/s) : min=641024, max=673792, per=100.05%, avg=665246.47, stdev=11746.95 cpu : usr=0.47%, sys=5.59%, ctx=5136, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.51%, 4=0.31%, 10=0.18% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=649MB/s, minb=665MB/s, maxb=665MB/s, mint=7885msec, maxt=7885msec Disk stats (read/write): sdb: ios=9969/0, merge=0/0, ticks=12815/0, in_queue=12816, util=96.17% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6638 read : io=5,120MB, bw=651MB/s, iops=650, runt= 7867msec clat (usec): min=1,401, max=10,016, avg=1529.66, stdev=252.22 bw (KB/s) : min=632832, max=675840, per=100.03%, avg=666610.20, stdev=12667.70 cpu : usr=0.25%, sys=4.84%, ctx=5131, majf=0, minf=279 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=5120/0, short=0/0 lat (msec): 2=99.59%, 4=0.18%, 10=0.21%, 20=0.02% Run status group 0 (all jobs): READ: io=5,120MB, aggrb=651MB/s, minb=666MB/s, maxb=666MB/s, mint=7867msec, maxt=7867msec Disk stats (read/write): sdb: ios=9989/0, merge=0/0, ticks=12881/0, in_queue=12883, util=96.60% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=6644 read : io=47,388MB, bw=790MB/s, iops=789, runt= 60001msec clat (usec): min=856, max=136K, avg=1259.44, stdev=1236.59 bw (KB/s) : min=372736, max=1001472, per=99.94%, avg=808243.76, stdev=100660.44 cpu : usr=0.57%, sys=7.56%, ctx=47587, majf=0, minf=324 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=47388/0, short=0/0 lat (usec): 1000=17.01% lat (msec): 2=82.79%, 4=0.04%, 10=0.14%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=47,388MB, aggrb=790MB/s, minb=809MB/s, maxb=809MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=94561/0, merge=0/0, ticks=97391/0, in_queue=97396, util=94.17% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6682 read : io=237MB, bw=4,042KB/s, iops=8,083, runt= 60001msec clat (usec): min=93, max=311K, avg=42241.75, stdev=3366.02 bw (KB/s) : min= 4, max= 4008, per=9.60%, avg=388.03, stdev=186.54 cpu : usr=1.30%, sys=3.85%, ctx=485263, majf=0, minf=116 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=485008/0, short=0/0 lat (usec): 100=1.21%, 250=98.29%, 500=0.11%, 750=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.08%, 20=0.02%, 50=0.01% lat (msec): 100=0.01%, 250=0.29%, 500=0.01% Run status group 0 (all jobs): READ: io=237MB, aggrb=4,041KB/s, minb=4,138KB/s, maxb=4,138KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=483271/0, merge=748/0, ticks=162192/0, in_queue=162274, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6692 read : io=226MB, bw=3,850KB/s, iops=7,700, runt= 60001msec clat (usec): min=92, max=275K, avg=30157.48, stdev=3007.98 bw (KB/s) : min= 6, max= 4051, per=4.50%, avg=173.34, stdev=25.99 cpu : usr=1.26%, sys=4.22%, ctx=462276, majf=0, minf=115 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=462042/0, short=0/0 lat (usec): 100=2.02%, 250=97.17%, 500=0.06%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.31%, 20=0.10%, 50=0.01% lat (msec): 100=0.01%, 250=0.33%, 500=0.01% Run status group 0 (all jobs): READ: io=226MB, aggrb=3,850KB/s, minb=3,942KB/s, maxb=3,942KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=459244/0, merge=1914/0, ticks=113993/0, in_queue=114071, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6702 read : io=155MB, bw=2,644KB/s, iops=5,287, runt= 60046msec clat (usec): min=73, max=521K, avg=742.66, stdev=6697.14 bw (KB/s) : min= 3, max= 1115, per=26.61%, avg=703.20, stdev=158.32 cpu : usr=1.36%, sys=3.54%, ctx=317864, majf=0, minf=4416 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=317519/0, short=0/0 lat (usec): 100=0.01%, 250=97.34%, 500=2.42%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.02%, 500=0.16%, 750=0.01% Run status group 0 (all jobs): READ: io=155MB, aggrb=2,643KB/s, minb=2,707KB/s, maxb=2,707KB/s, mint=60046msec, maxt=60046msec Disk stats (read/write): sdb: ios=316954/0, merge=0/0, ticks=232224/0, in_queue=232327, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6713 read : io=140MB, bw=2,391KB/s, iops=4,781, runt= 60059msec clat (usec): min=86, max=636K, avg=826.01, stdev=6998.38 bw (KB/s) : min= 0, max= 992, per=26.63%, avg=636.50, stdev=136.76 cpu : usr=1.06%, sys=3.31%, ctx=287430, majf=0, minf=6530 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=287143/0, short=0/0 lat (usec): 100=0.01%, 250=96.17%, 500=3.43%, 750=0.09%, 1000=0.03% lat (msec): 2=0.03%, 4=0.01%, 10=0.02%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.03%, 500=0.18%, 750=0.01% Run status group 0 (all jobs): READ: io=140MB, aggrb=2,390KB/s, minb=2,447KB/s, maxb=2,447KB/s, mint=60059msec, maxt=60059msec Disk stats (read/write): sdb: ios=286758/0, merge=0/0, ticks=233849/0, in_queue=233917, util=99.90% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6728 read : io=20,480MB, bw=501MB/s, iops=501, runt= 40838msec clat (msec): min=1, max=636, avg= 7.87, stdev=21.14 bw (KB/s) : min=12000, max=216654, per=27.17%, avg=139531.57, stdev=32999.93 cpu : usr=0.14%, sys=1.37%, ctx=20612, majf=0, minf=1129 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=75.32%, 4=22.48%, 10=0.22%, 20=0.01%, 250=0.04% lat (msec): 500=1.92%, 750=0.01% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=501MB/s, minb=514MB/s, maxb=514MB/s, mint=40838msec, maxt=40838msec Disk stats (read/write): sdb: ios=40922/0, merge=0/0, ticks=245506/0, in_queue=245520, util=99.95% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6738 read : io=20,480MB, bw=538MB/s, iops=538, runt= 38056msec clat (msec): min=1, max=313, avg= 7.08, stdev=19.81 bw (KB/s) : min=69228, max=320894, per=28.42%, avg=156639.41, stdev=35339.86 cpu : usr=0.12%, sys=1.45%, ctx=20593, majf=0, minf=1128 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=88.96%, 4=8.94%, 10=0.21%, 20=0.04%, 100=0.01% lat (msec): 250=0.10%, 500=1.74% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=538MB/s, minb=551MB/s, maxb=551MB/s, mint=38056msec, maxt=38056msec Disk stats (read/write): sdb: ios=40950/0, merge=0/0, ticks=215835/0, in_queue=215853, util=99.93% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6748 read : io=31,779MB, bw=530MB/s, iops=529, runt= 60005msec clat (msec): min=1, max=637, avg= 7.58, stdev=20.71 bw (KB/s) : min= 4544, max=237093, per=27.02%, avg=146544.61, stdev=34384.65 cpu : usr=0.11%, sys=1.19%, ctx=31954, majf=0, minf=1132 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=31779/0, short=0/0 lat (msec): 2=73.44%, 4=24.46%, 10=0.20%, 20=0.03%, 100=0.01% lat (msec): 250=0.02%, 500=1.85%, 750=0.01% Run status group 0 (all jobs): READ: io=31,779MB, aggrb=530MB/s, minb=542MB/s, maxb=542MB/s, mint=60005msec, maxt=60005msec Disk stats (read/write): sdb: ios=63438/0, merge=0/0, ticks=351975/0, in_queue=352840, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6758 read : io=20,480MB, bw=524MB/s, iops=524, runt= 39048msec clat (msec): min=1, max=554, avg= 7.50, stdev=20.47 bw (KB/s) : min= 2670, max=288820, per=27.37%, avg=146972.79, stdev=34329.33 cpu : usr=0.13%, sys=1.59%, ctx=20618, majf=0, minf=1126 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=86.56%, 4=11.36%, 10=0.20%, 50=0.01%, 100=0.01% lat (msec): 250=0.08%, 500=1.78%, 750=0.01% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=524MB/s, minb=537MB/s, maxb=537MB/s, mint=39048msec, maxt=39048msec Disk stats (read/write): sdb: ios=40640/0, merge=0/0, ticks=223670/0, in_queue=223694, util=99.93% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6768 read : io=20,480MB, bw=520MB/s, iops=520, runt= 39371msec clat (msec): min=1, max=566, avg= 7.27, stdev=20.09 bw (KB/s) : min= 3710, max=309248, per=28.37%, avg=151125.51, stdev=36097.46 cpu : usr=0.10%, sys=1.13%, ctx=20573, majf=0, minf=1126 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=20480/0, short=0/0 lat (msec): 2=80.87%, 4=16.97%, 10=0.20%, 20=0.03%, 100=0.01% lat (msec): 250=0.19%, 500=1.71%, 750=0.01% Run status group 0 (all jobs): READ: io=20,480MB, aggrb=520MB/s, minb=533MB/s, maxb=533MB/s, mint=39371msec, maxt=39371msec Disk stats (read/write): sdb: ios=40891/0, merge=0/0, ticks=233272/0, in_queue=233301, util=99.93% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=6778 read : io=33,322MB, bw=555MB/s, iops=555, runt= 60006msec clat (msec): min=1, max=587, avg= 6.73, stdev=20.18 bw (KB/s) : min=61275, max=288768, per=29.12%, avg=165603.66, stdev=39728.01 cpu : usr=0.10%, sys=1.15%, ctx=33465, majf=0, minf=1205 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=33322/0, short=0/0 lat (msec): 2=80.65%, 4=17.33%, 10=0.20%, 20=0.03%, 100=0.01% lat (msec): 250=0.01%, 500=1.76%, 750=0.01% Run status group 0 (all jobs): READ: io=33,322MB, aggrb=555MB/s, minb=569MB/s, maxb=569MB/s, mint=60006msec, maxt=60006msec Disk stats (read/write): sdb: ios=66519/0, merge=0/0, ticks=362934/0, in_queue=363713, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing read tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=6819 read : io=62,803KB, bw=1,047KB/s, iops=2,093, runt= 60006msec clat (usec): min=99, max=504K, avg=30636.40, stdev=3750.32 bw (KB/s) : min= 0, max= 40, per=1.52%, avg=15.90, stdev= 0.96 cpu : usr=0.00%, sys=0.01%, ctx=128813, majf=0, minf=2004 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=125605/0, short=0/0 lat (usec): 100=0.90%, 250=38.88%, 500=0.35%, 750=0.01%, 1000=0.01% lat (msec): 10=2.24%, 20=2.50%, 50=17.43%, 100=36.72%, 250=0.92% lat (msec): 500=0.05%, 750=0.01% Run status group 0 (all jobs): READ: io=62,802KB, aggrb=1,046KB/s, minb=1,071KB/s, maxb=1,071KB/s, mint=60006msec, maxt=60006msec Disk stats (read/write): sdb: ios=57373/0, merge=67898/0, ticks=406772/0, in_queue=406875, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=6889 read : io=171MB, bw=2,923KB/s, iops=5,846, runt= 60010msec clat (usec): min=93, max=504K, avg=61724.80, stdev=3405.84 bw (KB/s) : min= 0, max= 3039, per=0.26%, avg= 7.64, stdev= 0.75 cpu : usr=0.05%, sys=0.16%, ctx=352541, majf=0, minf=1875 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=350847/0, short=0/0 lat (usec): 100=1.74%, 250=86.50%, 500=0.20%, 750=0.01%, 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.48%, 20=0.48%, 50=3.40% lat (msec): 100=1.18%, 250=5.92%, 500=0.06%, 750=0.01% Run status group 0 (all jobs): READ: io=171MB, aggrb=2,923KB/s, minb=2,993KB/s, maxb=2,993KB/s, mint=60010msec, maxt=60010msec Disk stats (read/write): sdb: ios=313402/0, merge=36593/0, ticks=408897/0, in_queue=409125, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=6959 read : io=3,941MB, bw=67,256KB/s, iops=135K, runt= 60003msec clat (usec): min=47, max=210K, avg=470.78, stdev=95.43 bw (KB/s) : min= 137, max= 2774, per=1.55%, avg=1039.48, stdev=20.07 cpu : usr=2.39%, sys=3.42%, ctx=8110186, majf=0, minf=66727 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=8071075/0, short=0/0 lat (usec): 50=0.01%, 100=0.01%, 250=7.05%, 500=57.06%, 750=32.31% lat (usec): 1000=2.98% lat (msec): 2=0.48%, 4=0.04%, 10=0.07%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=3,941MB, aggrb=67,255KB/s, minb=68,869KB/s, maxb=68,869KB/s, mint=60003msec, maxt=60003msec Disk stats (read/write): sdb: ios=8070653/0, merge=0/0, ticks=3541891/0, in_queue=3541944, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7029 read : io=3,063MB, bw=52,281KB/s, iops=105K, runt= 60001msec clat (usec): min=95, max=210K, avg=606.40, stdev=121.09 bw (KB/s) : min= 83, max= 2297, per=1.56%, avg=814.49, stdev=19.85 cpu : usr=1.86%, sys=2.56%, ctx=6300174, majf=0, minf=78559 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=6273792/0, short=0/0 lat (usec): 100=0.01%, 250=2.36%, 500=28.68%, 750=52.21%, 1000=14.42% lat (msec): 2=2.07%, 4=0.16%, 10=0.08%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): READ: io=3,063MB, aggrb=52,280KB/s, minb=53,535KB/s, maxb=53,535KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=6273039/0, merge=4/0, ticks=3599430/0, in_queue=3599682, util=99.89% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7100 read : io=33,845MB, bw=563MB/s, iops=563, runt= 60080msec clat (msec): min=1, max=6,661, avg=121.15, stdev=101.76 bw (KB/s) : min= 158, max=106283, per=1.46%, avg=8405.01, stdev=454.65 cpu : usr=0.01%, sys=0.09%, ctx=34092, majf=0, minf=18415 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=33845/0, short=0/0 lat (msec): 2=96.38%, 4=1.49%, 10=0.19%, 20=0.01%, 100=0.01% lat (msec): 250=0.01%, 500=0.01%, 750=0.01%, 1000=0.01%, 2000=0.06% lat (msec): >=2000=1.82% Run status group 0 (all jobs): READ: io=33,845MB, aggrb=563MB/s, minb=577MB/s, maxb=577MB/s, mint=60080msec, maxt=60080msec Disk stats (read/write): sdb: ios=67450/0, merge=0/0, ticks=5450900/0, in_queue=5738883, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7171 read : io=34,179MB, bw=569MB/s, iops=568, runt= 60078msec clat (msec): min=1, max=6,780, avg=113.10, stdev=101.22 bw (KB/s) : min= 199, max=120000, per=1.78%, avg=10353.36, stdev=1151.29 cpu : usr=0.01%, sys=0.09%, ctx=34421, majf=0, minf=18195 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=34179/0, short=0/0 lat (msec): 2=95.15%, 4=1.95%, 10=0.24%, 20=0.04%, 50=0.29% lat (msec): 100=0.30%, 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.01% lat (msec): 2000=0.15%, >=2000=1.83% Run status group 0 (all jobs): READ: io=34,179MB, aggrb=569MB/s, minb=583MB/s, maxb=583MB/s, mint=60078msec, maxt=60078msec Disk stats (read/write): sdb: ios=68117/0, merge=0/0, ticks=5583123/0, in_queue=5857243, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=read --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=read, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7241 read : io=34,014MB, bw=566MB/s, iops=566, runt= 60080msec clat (msec): min=1, max=6,700, avg=114.55, stdev=101.83 bw (KB/s) : min= 154, max=111881, per=1.55%, avg=9006.52, stdev=457.28 cpu : usr=0.01%, sys=0.09%, ctx=34276, majf=0, minf=18443 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=34014/0, short=0/0 lat (msec): 2=96.37%, 4=1.49%, 10=0.20%, 20=0.01%, 100=0.01% lat (msec): 250=0.01%, 500=0.01%, 750=0.02%, 1000=0.01%, 2000=0.06% lat (msec): >=2000=1.81% Run status group 0 (all jobs): READ: io=34,014MB, aggrb=566MB/s, minb=580MB/s, maxb=580MB/s, mint=60080msec, maxt=60080msec Disk stats (read/write): sdb: ios=67796/0, merge=0/0, ticks=5315833/0, in_queue=5600581, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7311 read : io=71,305MB, bw=1,187MB/s, iops=1,187, runt= 60051msec clat (msec): min=2, max=321, avg=54.20, stdev= 2.49 bw (KB/s) : min= 3631, max=27463, per=1.56%, avg=18949.62, stdev=346.33 cpu : usr=0.01%, sys=0.15%, ctx=71602, majf=0, minf=18167 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=71305/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=1.19%, 50=41.16%, 100=56.92% lat (msec): 250=0.62%, 500=0.10% Run status group 0 (all jobs): READ: io=71,305MB, aggrb=1,187MB/s, minb=1,216MB/s, maxb=1,216MB/s, mint=60051msec, maxt=60051msec Disk stats (read/write): sdb: ios=142159/0, merge=0/0, ticks=7617099/0, in_queue=7620791, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7382 read : io=72,040MB, bw=1,200MB/s, iops=1,199, runt= 60056msec clat (msec): min=3, max=326, avg=53.59, stdev= 2.38 bw (KB/s) : min= 3580, max=27203, per=1.56%, avg=19165.55, stdev=314.24 cpu : usr=0.01%, sys=0.16%, ctx=72416, majf=0, minf=18827 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=72040/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=1.26%, 50=41.97%, 100=56.34% lat (msec): 250=0.32%, 500=0.10% Run status group 0 (all jobs): READ: io=72,040MB, aggrb=1,200MB/s, minb=1,228MB/s, maxb=1,228MB/s, mint=60056msec, maxt=60056msec Disk stats (read/write): sdb: ios=143647/0, merge=0/0, ticks=7616237/0, in_queue=7620176, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randread --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randread, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7452 read : io=75,976MB, bw=1,265MB/s, iops=1,265, runt= 60044msec clat (msec): min=2, max=325, avg=51.29, stdev= 2.51 bw (KB/s) : min= 3710, max=29767, per=1.55%, avg=20025.71, stdev=440.68 cpu : usr=0.02%, sys=0.17%, ctx=76338, majf=0, minf=18715 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=75976/0, short=0/0 lat (msec): 4=0.01%, 10=0.01%, 20=2.17%, 50=50.35%, 100=45.98% lat (msec): 250=1.42%, 500=0.07% Run status group 0 (all jobs): READ: io=75,976MB, aggrb=1,265MB/s, minb=1,296MB/s, maxb=1,296MB/s, mint=60044msec, maxt=60044msec Disk stats (read/write): sdb: ios=151467/0, merge=0/0, ticks=7613441/0, in_queue=7616854, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=1 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7553 write: io=424MB, bw=7,243KB/s, iops=14,486, runt= 60001msec clat (usec): min=40, max=135K, avg=64.51, stdev=269.22 bw (KB/s) : min= 1781, max= 7532, per=100.05%, avg=7246.92, stdev=575.30 cpu : usr=13.71%, sys=35.07%, ctx=869448, majf=0, minf=30 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/869212, short=0/0 lat (usec): 50=18.02%, 100=73.43%, 250=8.51%, 500=0.03%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): WRITE: io=424MB, aggrb=7,243KB/s, minb=7,417KB/s, maxb=7,417KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/867581, merge=0/0, ticks=0/44955, in_queue=45039, util=75.23% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7560 write: io=452MB, bw=7,715KB/s, iops=15,430, runt= 60001msec clat (usec): min=40, max=135K, avg=60.25, stdev=262.93 bw (KB/s) : min= 4666, max= 8094, per=100.11%, avg=7723.35, stdev=496.96 cpu : usr=13.44%, sys=35.98%, ctx=926104, majf=0, minf=25 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/925868, short=0/0 lat (usec): 50=16.98%, 100=79.19%, 250=3.81%, 500=0.01%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=452MB, aggrb=7,715KB/s, minb=7,900KB/s, maxb=7,900KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/924188, merge=0/0, ticks=0/45179, in_queue=45275, util=75.62% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7567 write: io=151MB, bw=2,581KB/s, iops=5,162, runt= 60001msec clat (usec): min=68, max=135K, avg=187.41, stdev=491.85 bw (KB/s) : min= 1168, max= 2760, per=100.08%, avg=2582.94, stdev=204.00 cpu : usr=4.72%, sys=12.81%, ctx=310086, majf=0, minf=2757 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/309770, short=0/0 lat (usec): 100=0.04%, 250=95.57%, 500=3.66%, 750=0.65%, 1000=0.03% lat (msec): 2=0.02%, 4=0.01%, 10=0.03%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=151MB, aggrb=2,581KB/s, minb=2,643KB/s, maxb=2,643KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/308929, merge=0/0, ticks=0/53389, in_queue=53363, util=89.15% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=30012289843 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7575 write: io=131MB, bw=2,241KB/s, iops=4,481, runt= 60001msec clat (usec): min=49, max=135K, avg=216.84, stdev=529.41 bw (KB/s) : min= 493, max= 2419, per=100.11%, avg=2242.39, stdev=221.57 cpu : usr=3.33%, sys=9.70%, ctx=269130, majf=0, minf=1043 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/268921, short=0/0 lat (usec): 50=0.01%, 100=0.01%, 250=92.25%, 500=6.19%, 750=1.31% lat (usec): 1000=0.10% lat (msec): 2=0.05%, 4=0.01%, 10=0.06%, 20=0.01%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=131MB, aggrb=2,240KB/s, minb=2,294KB/s, maxb=2,294KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/268165, merge=0/0, ticks=0/55239, in_queue=55213, util=92.22% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7582 write: io=5,120MB, bw=548MB/s, iops=548, runt= 9338msec clat (msec): min=1, max=17, avg= 1.82, stdev= 1.52 bw (KB/s) : min=502802, max=604998, per=99.74%, avg=559979.83, stdev=37884.70 cpu : usr=1.15%, sys=3.64%, ctx=5134, majf=0, minf=23 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=83.12%, 4=9.82%, 10=6.25%, 20=0.80% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=548MB/s, minb=561MB/s, maxb=561MB/s, mint=9338msec, maxt=9338msec Disk stats (read/write): sdb: ios=0/10051, merge=0/0, ticks=0/14634, in_queue=14636, util=96.56% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7588 write: io=5,120MB, bw=552MB/s, iops=552, runt= 9272msec clat (msec): min=1, max=18, avg= 1.80, stdev= 1.54 bw (KB/s) : min=506890, max=606208, per=99.87%, avg=564689.72, stdev=35242.83 cpu : usr=0.19%, sys=2.02%, ctx=5134, majf=0, minf=23 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=84.26%, 4=10.04%, 10=4.88%, 20=0.82% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=552MB/s, minb=565MB/s, maxb=565MB/s, mint=9272msec, maxt=9272msec Disk stats (read/write): sdb: ios=0/10123, merge=0/0, ticks=0/15113, in_queue=15115, util=97.86% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7595 write: io=32,919MB, bw=549MB/s, iops=548, runt= 60001msec clat (msec): min=1, max=135, avg= 1.82, stdev= 2.07 bw (KB/s) : min=252517, max=628736, per=100.15%, avg=562651.38, stdev=50651.98 cpu : usr=0.69%, sys=2.89%, ctx=33010, majf=0, minf=26 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/32919, short=0/0 lat (msec): 2=84.16%, 4=9.50%, 10=5.51%, 20=0.81%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=32,919MB, aggrb=549MB/s, minb=562MB/s, maxb=562MB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/65704, merge=0/0, ticks=0/95875, in_queue=95888, util=97.96% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7602 write: io=5,120MB, bw=516MB/s, iops=516, runt= 9918msec clat (msec): min=1, max=19, avg= 1.93, stdev= 1.67 bw (KB/s) : min=468992, max=610304, per=99.76%, avg=527331.00, stdev=32182.19 cpu : usr=0.51%, sys=3.62%, ctx=5143, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=81.82%, 4=7.87%, 10=9.49%, 20=0.82% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=516MB/s, minb=529MB/s, maxb=529MB/s, mint=9918msec, maxt=9918msec Disk stats (read/write): sdb: ios=0/9978, merge=0/0, ticks=0/15741, in_queue=15747, util=96.39% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7608 write: io=5,120MB, bw=508MB/s, iops=508, runt= 10073msec clat (msec): min=1, max=18, avg= 1.96, stdev= 1.65 bw (KB/s) : min=464186, max=573195, per=100.05%, avg=520723.00, stdev=34475.38 cpu : usr=0.37%, sys=2.86%, ctx=5136, majf=0, minf=22 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/5120, short=0/0 lat (msec): 2=80.06%, 4=9.49%, 10=9.67%, 20=0.78% Run status group 0 (all jobs): WRITE: io=5,120MB, aggrb=508MB/s, minb=520MB/s, maxb=520MB/s, mint=10073msec, maxt=10073msec Disk stats (read/write): sdb: ios=0/10086, merge=0/0, ticks=0/16217, in_queue=16222, util=97.47% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=300122898432 --runtime=60 --numjobs=1 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 1 process file1: (groupid=0, jobs=1): err= 0: pid=7614 write: io=26,271MB, bw=438MB/s, iops=437, runt= 60002msec clat (msec): min=1, max=135, avg= 2.28, stdev= 2.81 bw (KB/s) : min=106071, max=546816, per=100.11%, avg=448855.08, stdev=46485.36 cpu : usr=0.39%, sys=2.45%, ctx=26365, majf=0, minf=77 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/26271, short=0/0 lat (msec): 2=80.46%, 4=6.94%, 10=11.32%, 20=1.22%, 50=0.04% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=26,271MB, aggrb=438MB/s, minb=448MB/s, maxb=448MB/s, mint=60002msec, maxt=60002msec Disk stats (read/write): sdb: ios=0/52438, merge=0/0, ticks=0/100180, in_queue=100190, util=98.36% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=4 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7622 write: io=943MB, bw=16,101KB/s, iops=32,202, runt= 60001msec clat (usec): min=38, max=159K, avg=119.25, stdev=154.31 bw (KB/s) : min= 2519, max= 5644, per=25.04%, avg=4031.93, stdev=290.27 cpu : usr=13.13%, sys=14.55%, ctx=1934980, majf=0, minf=112 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/1932211, short=0/0 lat (usec): 50=0.77%, 100=40.75%, 250=58.15%, 500=0.31%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): WRITE: io=943MB, aggrb=16,101KB/s, minb=16,487KB/s, maxb=16,487KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/1929247, merge=0/0, ticks=0/177209, in_queue=177121, util=98.25% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7632 write: io=1,201MB, bw=20,490KB/s, iops=40,979, runt= 60001msec clat (usec): min=37, max=168K, avg=92.60, stdev=167.69 bw (KB/s) : min= 3209, max= 6623, per=25.08%, avg=5137.78, stdev=405.84 cpu : usr=17.61%, sys=17.69%, ctx=2462448, majf=0, minf=100 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/2458813, short=0/0 lat (usec): 50=1.31%, 100=72.49%, 250=26.09%, 500=0.10%, 750=0.01% lat (usec): 1000=0.01% lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=0.01% lat (msec): 250=0.01% Run status group 0 (all jobs): WRITE: io=1,201MB, aggrb=20,489KB/s, minb=20,981KB/s, maxb=20,981KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/2455303, merge=0/0, ticks=0/153458, in_queue=153278, util=96.31% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7672 write: io=483MB, bw=8,238KB/s, iops=16,475, runt= 60009msec clat (usec): min=55, max=135K, avg=236.14, stdev=389.29 bw (KB/s) : min= 461, max= 2388, per=25.03%, avg=2061.70, stdev=97.14 cpu : usr=6.36%, sys=8.94%, ctx=993480, majf=0, minf=7989 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/988669, short=0/0 lat (usec): 100=0.01%, 250=88.91%, 500=8.24%, 750=2.42%, 1000=0.14% lat (msec): 2=0.06%, 4=0.02%, 10=0.09%, 20=0.10%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=483MB, aggrb=8,237KB/s, minb=8,435KB/s, maxb=8,435KB/s, mint=60009msec, maxt=60009msec Disk stats (read/write): sdb: ios=0/988341, merge=0/0, ticks=0/205985, in_queue=205696, util=99.58% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=30012289843 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7685 write: io=422MB, bw=7,197KB/s, iops=14,393, runt= 60001msec clat (usec): min=46, max=135K, avg=270.93, stdev=361.39 bw (KB/s) : min= 359, max= 2092, per=25.03%, avg=1801.48, stdev=86.10 cpu : usr=6.31%, sys=8.93%, ctx=867864, majf=0, minf=15854 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/863635, short=0/0 lat (usec): 50=0.01%, 100=0.01%, 250=79.49%, 500=14.78%, 750=4.89% lat (usec): 1000=0.43% lat (msec): 2=0.14%, 4=0.03%, 10=0.12%, 20=0.11%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=422MB, aggrb=7,196KB/s, minb=7,369KB/s, maxb=7,369KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/863062, merge=0/0, ticks=0/206906, in_queue=206651, util=99.60% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7695 write: io=20,480MB, bw=621MB/s, iops=620, runt= 32997msec clat (msec): min=1, max=28, avg= 6.44, stdev= 1.36 bw (KB/s) : min=136942, max=210522, per=25.02%, avg=159044.11, stdev=4871.67 cpu : usr=0.24%, sys=1.19%, ctx=20583, majf=0, minf=98 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.04%, 4=17.03%, 10=76.35%, 20=6.15%, 50=0.43% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=621MB/s, minb=636MB/s, maxb=636MB/s, mint=32997msec, maxt=32997msec Disk stats (read/write): sdb: ios=0/40699, merge=0/0, ticks=0/239534, in_queue=239558, util=99.88% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7705 write: io=20,480MB, bw=631MB/s, iops=630, runt= 32478msec clat (msec): min=1, max=29, avg= 6.34, stdev= 1.33 bw (KB/s) : min=140749, max=201142, per=25.02%, avg=161575.39, stdev=5065.88 cpu : usr=0.27%, sys=1.18%, ctx=20559, majf=0, minf=95 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.03%, 4=17.67%, 10=76.52%, 20=5.40%, 50=0.39% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=631MB/s, minb=646MB/s, maxb=646MB/s, mint=32478msec, maxt=32478msec Disk stats (read/write): sdb: ios=0/40738, merge=0/0, ticks=0/235833, in_queue=235857, util=99.88% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7714 write: io=37,722MB, bw=629MB/s, iops=628, runt= 60006msec clat (msec): min=1, max=135, avg= 6.36, stdev= 1.81 bw (KB/s) : min=106064, max=209269, per=25.04%, avg=161165.27, stdev=6808.76 cpu : usr=0.24%, sys=1.15%, ctx=37862, majf=0, minf=111 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/37722, short=0/0 lat (msec): 2=0.01%, 4=18.41%, 10=75.23%, 20=5.98%, 50=0.33% lat (msec): 100=0.02%, 250=0.03% Run status group 0 (all jobs): WRITE: io=37,722MB, aggrb=629MB/s, minb=644MB/s, maxb=644MB/s, mint=60006msec, maxt=60006msec Disk stats (read/write): sdb: ios=0/75305, merge=0/0, ticks=0/439296, in_queue=439325, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7725 write: io=20,480MB, bw=571MB/s, iops=570, runt= 35868msec clat (usec): min=972, max=190K, avg=6993.65, stdev=2683.67 bw (KB/s) : min=79364, max=206848, per=25.06%, avg=146493.09, stdev=10604.52 cpu : usr=0.22%, sys=1.11%, ctx=20570, majf=0, minf=97 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (usec): 1000=0.01% lat (msec): 2=0.04%, 4=16.97%, 10=72.08%, 20=10.28%, 50=0.39% lat (msec): 100=0.16%, 250=0.07% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=571MB/s, minb=585MB/s, maxb=585MB/s, mint=35868msec, maxt=35868msec Disk stats (read/write): sdb: ios=0/40869, merge=0/0, ticks=0/263564, in_queue=263582, util=99.92% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7735 write: io=20,480MB, bw=532MB/s, iops=531, runt= 38526msec clat (msec): min=1, max=126, avg= 7.51, stdev= 2.29 bw (KB/s) : min=39083, max=178176, per=25.01%, avg=136149.95, stdev=12446.65 cpu : usr=0.19%, sys=1.03%, ctx=20562, majf=0, minf=97 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/20480, short=0/0 lat (msec): 2=0.02%, 4=13.36%, 10=70.36%, 20=15.06%, 50=1.07% lat (msec): 100=0.07%, 250=0.05% Run status group 0 (all jobs): WRITE: io=20,480MB, aggrb=532MB/s, minb=544MB/s, maxb=544MB/s, mint=38526msec, maxt=38526msec Disk stats (read/write): sdb: ios=0/40662, merge=0/0, ticks=0/280660, in_queue=280686, util=99.95% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=300122898432 --runtime=60 --numjobs=4 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 4 processes file1: (groupid=0, jobs=4): err= 0: pid=7744 write: io=34,194MB, bw=570MB/s, iops=569, runt= 60004msec clat (msec): min=1, max=192, avg= 7.01, stdev= 3.21 bw (KB/s) : min=61832, max=201972, per=25.04%, avg=146129.52, stdev=8871.16 cpu : usr=0.21%, sys=1.07%, ctx=34349, majf=0, minf=132 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/34194, short=0/0 lat (msec): 2=0.01%, 4=28.86%, 10=58.79%, 20=11.38%, 50=0.58% lat (msec): 100=0.27%, 250=0.12% Run status group 0 (all jobs): WRITE: io=34,194MB, aggrb=570MB/s, minb=584MB/s, maxb=584MB/s, mint=60004msec, maxt=60004msec Disk stats (read/write): sdb: ios=0/68279, merge=0/0, ticks=0/443635, in_queue=443683, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ ################################################################################# Executing write tests with numjobs=64 --------------------------------------------------------------------------------- Executing test 1: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7756 write: io=5,634MB, bw=96,148KB/s, iops=192K, runt= 60001msec clat (usec): min=45, max=236K, avg=300.70, stdev=74.86 bw (KB/s) : min= 90, max= 2237, per=1.69%, avg=1626.07, stdev=21.04 cpu : usr=3.09%, sys=5.49%, ctx=11568538, majf=0, minf=1813 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/11537949, short=0/0 lat (usec): 50=0.01%, 100=0.34%, 250=38.75%, 500=55.04%, 750=3.53% lat (usec): 1000=0.88% lat (msec): 2=0.95%, 4=0.33%, 10=0.16%, 20=0.03%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=5,634MB, aggrb=96,147KB/s, minb=98,455KB/s, maxb=98,455KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/9826306, merge=0/1536401, ticks=0/2963995, in_queue=2961952, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 2: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7855 write: io=5,620MB, bw=95,906KB/s, iops=192K, runt= 60001msec clat (usec): min=41, max=218K, avg=277.85, stdev=72.96 bw (KB/s) : min= 154, max= 2214, per=1.84%, avg=1767.99, stdev=25.22 cpu : usr=3.04%, sys=5.42%, ctx=11539726, majf=0, minf=1808 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/11508908, short=0/0 lat (usec): 50=0.01%, 100=0.29%, 250=38.88%, 500=54.67%, 750=3.69% lat (usec): 1000=0.93% lat (msec): 2=1.03%, 4=0.33%, 10=0.15%, 20=0.02%, 50=0.01% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=5,620MB, aggrb=95,905KB/s, minb=98,207KB/s, maxb=98,207KB/s, mint=60001msec, maxt=60001msec Disk stats (read/write): sdb: ios=0/9879170, merge=0/1454770, ticks=0/2978387, in_queue=2976495, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 3: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7925 write: io=1,658MB, bw=28,284KB/s, iops=56,568, runt= 60016msec clat (usec): min=49, max=208K, avg=1149.37, stdev=302.89 bw (KB/s) : min= 71, max= 1928, per=1.52%, avg=430.28, stdev=11.80 cpu : usr=1.08%, sys=1.69%, ctx=3502227, majf=0, minf=45403 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/3395029, short=0/0 lat (usec): 50=0.01%, 100=0.01%, 250=3.99%, 500=21.92%, 750=27.45% lat (usec): 1000=21.69% lat (msec): 2=20.58%, 4=1.76%, 10=0.57%, 20=1.98%, 50=0.07% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=1,658MB, aggrb=28,284KB/s, minb=28,963KB/s, maxb=28,963KB/s, mint=60016msec, maxt=60016msec Disk stats (read/write): sdb: ios=0/3394885, merge=0/0, ticks=0/3655693, in_queue=3655832, util=99.82% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 4: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=512 --size=30012289843 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=512-512/512-512, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=7996 write: io=1,458MB, bw=24,870KB/s, iops=49,740, runt= 60012msec clat (usec): min=75, max=207K, avg=1294.76, stdev=244.97 bw (KB/s) : min= 65, max= 2019, per=1.54%, avg=382.94, stdev= 7.01 cpu : usr=1.00%, sys=1.43%, ctx=3017721, majf=0, minf=94439 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/2985006, short=0/0 lat (usec): 100=0.01%, 250=2.00%, 500=11.21%, 750=17.40%, 1000=20.95% lat (msec): 2=40.72%, 4=4.74%, 10=2.07%, 20=0.83%, 50=0.07% lat (msec): 100=0.01%, 250=0.01% Run status group 0 (all jobs): WRITE: io=1,458MB, aggrb=24,870KB/s, minb=25,466KB/s, maxb=25,466KB/s, mint=60012msec, maxt=60012msec Disk stats (read/write): sdb: ios=0/2984260, merge=0/0, ticks=0/3680523, in_queue=3680681, util=99.86% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 5: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8067 write: io=36,652MB, bw=610MB/s, iops=609, runt= 60095msec clat (msec): min=2, max=2,683, avg=95.04, stdev= 4.82 bw (KB/s) : min= 381, max=36790, per=1.74%, avg=10840.51, stdev=584.10 cpu : usr=0.02%, sys=0.05%, ctx=36825, majf=0, minf=1953 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/36652, short=0/0 lat (msec): 4=0.01%, 10=0.02%, 20=0.35%, 50=6.40%, 100=31.81% lat (msec): 250=59.69%, 500=1.57%, 750=0.05%, 1000=0.02%, 2000=0.05% lat (msec): >=2000=0.03% Run status group 0 (all jobs): WRITE: io=36,652MB, aggrb=610MB/s, minb=625MB/s, maxb=625MB/s, mint=60095msec, maxt=60095msec Disk stats (read/write): sdb: ios=0/73026, merge=0/0, ticks=0/7558591, in_queue=7565050, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 6: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8137 write: io=32,900MB, bw=547MB/s, iops=547, runt= 60097msec clat (msec): min=1, max=2,742, avg=123.21, stdev=12.56 bw (KB/s) : min= 373, max=35859, per=1.53%, avg=8594.33, stdev=212.46 cpu : usr=0.02%, sys=0.05%, ctx=33039, majf=0, minf=1681 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/32900, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.04%, 50=2.93% lat (msec): 100=25.95%, 250=70.25%, 500=0.66%, 750=0.03%, 1000=0.01% lat (msec): 2000=0.06%, >=2000=0.04% Run status group 0 (all jobs): WRITE: io=32,900MB, aggrb=547MB/s, minb=561MB/s, maxb=561MB/s, mint=60097msec, maxt=60097msec Disk stats (read/write): sdb: ios=0/65502, merge=0/0, ticks=0/7597093, in_queue=7605468, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 7: ./fio --filename=/dev/sdb --direct=1 --rw=write --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8207 write: io=36,686MB, bw=610MB/s, iops=610, runt= 60093msec clat (msec): min=1, max=2,503, avg=93.25, stdev= 9.54 bw (KB/s) : min= 409, max=33412, per=1.79%, avg=11177.12, stdev=624.64 cpu : usr=0.02%, sys=0.05%, ctx=36838, majf=0, minf=1803 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/36686, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.02%, 20=0.49%, 50=33.56% lat (msec): 100=24.39%, 250=30.86%, 500=10.54%, 750=0.02%, 1000=0.02% lat (msec): 2000=0.06%, >=2000=0.03% Run status group 0 (all jobs): WRITE: io=36,686MB, aggrb=610MB/s, minb=625MB/s, maxb=625MB/s, mint=60093msec, maxt=60093msec Disk stats (read/write): sdb: ios=0/73099, merge=0/0, ticks=0/7431715, in_queue=7438065, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 8: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8279 write: io=33,290MB, bw=554MB/s, iops=553, runt= 60112msec clat (msec): min=1, max=543, avg=116.42, stdev= 8.73 bw (KB/s) : min= 1885, max=20277, per=1.57%, avg=8917.39, stdev=279.71 cpu : usr=0.01%, sys=0.05%, ctx=33389, majf=0, minf=1591 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/33290, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.22%, 50=15.62% lat (msec): 100=32.73%, 250=46.33%, 500=5.03%, 750=0.04% Run status group 0 (all jobs): WRITE: io=33,290MB, aggrb=554MB/s, minb=567MB/s, maxb=567MB/s, mint=60112msec, maxt=60112msec Disk stats (read/write): sdb: ios=0/66308, merge=0/0, ticks=0/7560417, in_queue=7569101, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 9: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=5G --runtime=60 --offset=294754189312 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8356 write: io=33,916MB, bw=564MB/s, iops=564, runt= 60119msec clat (msec): min=1, max=540, avg=109.53, stdev= 8.27 bw (KB/s) : min= 2003, max=19922, per=1.63%, avg=9435.49, stdev=282.10 cpu : usr=0.02%, sys=0.05%, ctx=34047, majf=0, minf=1987 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/33916, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.03%, 20=0.23%, 50=16.05% lat (msec): 100=32.97%, 250=46.49%, 500=4.21%, 750=0.01% Run status group 0 (all jobs): WRITE: io=33,916MB, aggrb=564MB/s, minb=578MB/s, maxb=578MB/s, mint=60119msec, maxt=60119msec Disk stats (read/write): sdb: ios=0/67556, merge=0/0, ticks=0/7575805, in_queue=7585107, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------ --------------------------------------------------------------------------------- Executing test 10: ./fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=1m --size=300122898432 --runtime=60 --numjobs=64 --group_reporting --name=file1 file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 ... file1: (g=0): rw=randwrite, bs=1M-1M/1M-1M, ioengine=sync, iodepth=1 Starting 64 processes file1: (groupid=0, jobs=64): err= 0: pid=8427 write: io=31,198MB, bw=519MB/s, iops=518, runt= 60121msec clat (msec): min=1, max=560, avg=123.37, stdev= 8.26 bw (KB/s) : min= 2048, max=18358, per=1.57%, avg=8336.44, stdev=284.99 cpu : usr=0.01%, sys=0.05%, ctx=31326, majf=0, minf=1592 IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued r/w: total=0/31198, short=0/0 lat (msec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.41%, 50=11.94% lat (msec): 100=28.80%, 250=54.97%, 500=3.81%, 750=0.04% Run status group 0 (all jobs): WRITE: io=31,198MB, aggrb=519MB/s, minb=531MB/s, maxb=531MB/s, mint=60121msec, maxt=60121msec Disk stats (read/write): sdb: ios=0/62123, merge=0/0, ticks=0/7591290, in_queue=7600245, util=100.00% Sleeping for 30 seconds... ------------------------------------------------------