Меряем производительность накопителей или снова про IOPS

Содержимое

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=64
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4252K/0K /s] [1063 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3745
read: io=131544KB, bw=4375.1KB/s, iops=1093, runt= 30061msec
slat (usec): min=3, max=33, avg= 5.40, stdev= 2.58
clat (msec): min=1, max=358, avg=58.49, stdev=33.30
lat (msec): min=1, max=358, avg=58.50, stdev=33.30
clat percentiles (msec):
| 1.00th=[ 4], 5.00th=[ 8], 10.00th=[ 14], 20.00th=[ 25],
| 30.00th=[ 36], 40.00th=[ 47], 50.00th=[ 59], 60.00th=[ 70],
| 70.00th=[ 81], 80.00th=[ 92], 90.00th=[ 103], 95.00th=[ 112],
| 99.00th=[ 122], 99.50th=[ 125], 99.90th=[ 157], 99.95th=[ 198],
| 99.99th=[ 289]
bw (KB/s): min= 3752, max= 5024, per=99.93%, avg=4372.10, stdev=221.75
lat (msec): 2=0.01%, 4=1.90%, 10=5.34%, 20=8.70%, 50=26.80%
lat (msec): 100=44.58%, 250=12.65%, 500=0.02%
cpu: usr=0.60%, sys=1.54%, ctx=32894, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued: total=r=32886/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=131544KB, aggrb=4375KB/s, minb=4375KB/s, maxb=4375KB/s, mint=30061msec, maxt=30061msec

Disk stats (read/write):
sdb: ios=32731/0, merge=1/0, ticks=1911326/0, in_queue=1913834, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4112K/0K /s] [1028 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3764
read: io=126804KB, bw=4222.4KB/s, iops=1055, runt= 30032msec
slat (usec): min=3, max=40, avg= 4.54, stdev= 2.16
clat (msec): min=1, max=78, avg=30.31, stdev=16.37
lat (msec): min=1, max=78, avg=30.31, stdev=16.37
clat percentiles (usec):
| 1.00th=[ 2864], 5.00th=[ 5024], 10.00th=[ 7840], 20.00th=[13504],
| 30.00th=[19072], 40.00th=[24704], 50.00th=[30080], 60.00th=[35584],
| 70.00th=[41216], 80.00th=[46336], 90.00th=[52480], 95.00th=[56576],
| 99.00th=[62208], 99.50th=[63744], 99.90th=[68096], 99.95th=[70144],
| 99.99th=[75264]
bw (KB/s): min= 3904, max= 4808, per=100.00%, avg=4222.38, stdev=211.15
lat (msec): 2=0.01%, 4=3.33%, 10=10.52%, 20=17.80%, 50=54.29%
lat (msec): 100=14.06%
cpu: usr=0.68%, sys=1.01%, ctx=31715, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=31701/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=126804KB, aggrb=4222KB/s, minb=4222KB/s, maxb=4222KB/s, mint=30032msec, maxt=30032msec

Disk stats (read/write):
sdb: ios=31575/0, merge=0/0, ticks=956196/0, in_queue=956760, util=99.70%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [3976K/0K /s] [994 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3771
read: io=122984KB, bw=4097.2KB/s, iops=1024, runt= 30017msec
slat (usec): min=3, max=34, avg= 4.53, stdev= 2.15
clat (msec): min=1, max=40, avg=15.61, stdev= 7.83
lat (msec): min=1, max=40, avg=15.62, stdev= 7.83
clat percentiles (usec):
| 1.00th=[ 2736], 5.00th=[ 3760], 10.00th=[ 5024], 20.00th=[ 7648],
| 30.00th=[10304], 40.00th=[12864], 50.00th=[15424], 60.00th=[18048],
| 70.00th=[20608], 80.00th=[23168], 90.00th=[26240], 95.00th=[28288],
| 99.00th=[32128], 99.50th=[33536], 99.90th=[36096], 99.95th=[37120],
| 99.99th=[39168]
bw (KB/s): min= 3944, max= 4728, per=100.00%, avg=4103.32, stdev=220.20
lat (msec): 2=0.01%, 4=6.07%, 10=22.92%, 20=39.08%, 50=31.93%
cpu: usr=0.59%, sys=1.15%, ctx=30756, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=30746/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=122984KB, aggrb=4097KB/s, minb=4097KB/s, maxb=4097KB/s, mint=30017msec, maxt=30017msec

Disk stats (read/write):
sdb: ios=30641/0, merge=0/0, ticks=478130/0, in_queue=478256, util=99.70%е

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [3888K/0K /s] [972 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3778
read: io=120648KB, bw=4020.6KB/s, iops=1005, runt= 30008msec
slat (usec): min=3, max=124, avg= 5.39, stdev= 3.01
clat (msec): min=1, max=22, avg= 7.95, stdev= 3.37
lat (msec): min=1, max=22, avg= 7.96, stdev= 3.37
clat percentiles (usec):
| 1.00th=[ 2608], 5.00th=[ 2992], 10.00th=[ 3504], 20.00th=[ 4512],
| 30.00th=[ 5728], 40.00th=[ 6816], 50.00th=[ 7840], 60.00th=[ 8768],
| 70.00th=[ 9920], 80.00th=[11072], 90.00th=[12480], 95.00th=[13632],
| 99.00th=[15936], 99.50th=[16768], 99.90th=[18304], 99.95th=[18816],
| 99.99th=[20096]
bw (KB/s): min= 3872, max= 4640, per=100.00%, avg=4026.14, stdev=215.32
lat (msec): 2=0.01%, 4=13.98%, 10=57.35%, 20=28.66%, 50=0.01%
cpu: usr=0.96%, sys=1.63%, ctx=30168, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=30162/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=120648KB, aggrb=4020KB/s, minb=4020KB/s, maxb=4020KB/s, mint=30008msec, maxt=30008msec

Disk stats (read/write):
sdb: ios=30068/0, merge=0/0, ticks=238837/0, in_queue=238884, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [3860K/0K /s] [965 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3785
read: io=119328KB, bw=3977.8KB/s, iops=994, runt= 30004msec
slat (usec): min=3, max=118, avg= 4.38, stdev= 2.36
clat (usec): min=1057, max=9386, avg=4016.37, stdev=983.10
lat (usec): min=1094, max=9390, avg=4021.16, stdev=983.13
clat percentiles (usec):
| 1.00th=[ 2608], 5.00th=[ 2736], 10.00th=[ 2896], 20.00th=[ 3120],
| 30.00th=[ 3248], 40.00th=[ 3728], 50.00th=[ 3984], 60.00th=[ 4128],
| 70.00th=[ 4320], 80.00th=[ 4896], 90.00th=[ 5280], 95.00th=[ 5984],
| 99.00th=[ 7008], 99.50th=[ 7264], 99.90th=[ 8256], 99.95th=[ 8512],
| 99.99th=[ 9152]
bw (KB/s): min= 3824, max= 4584, per=100.00%, avg=3983.19, stdev=212.25
lat (msec): 2=0.01%, 4=51.13%, 10=48.86%
cpu: usr=0.54%, sys=1.12%, ctx=29841, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=29832/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=119328KB, aggrb=3977KB/s, minb=3977KB/s, maxb=3977KB/s, mint=30004msec, maxt=30004msec

Disk stats (read/write):
sdb: ios=29742/0, merge=0/0, ticks=119303/0, in_queue=119300, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [3752K/0K /s] [938 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3796
read: io=116604KB, bw=3886.7KB/s, iops=971, runt= 30001msec
slat (usec): min=4, max=133, avg= 7.87, stdev= 3.84
clat (usec): min=676, max=1644, avg=1018.74, stdev=103.74
lat (usec): min=691, max=1660, avg=1027.08, stdev=104.32
clat percentiles (usec):
| 1.00th=[ 844], 5.00th=[ 852], 10.00th=[ 868], 20.00th=[ 884],
| 30.00th=[ 980], 40.00th=[ 996], 50.00th=[ 1012], 60.00th=[ 1096],
| 70.00th=[ 1112], 80.00th=[ 1112], 90.00th=[ 1128], 95.00th=[ 1144],
| 99.00th=[ 1192], 99.50th=[ 1224], 99.90th=[ 1240], 99.95th=[ 1352],
| 99.99th=[ 1416]
bw (KB/s): min= 3704, max= 4408, per=100.00%, avg=3892.61, stdev=197.78
lat (usec): 750=0.56%, 1000=46.23%
lat (msec): 2=53.21%
cpu: usr=0.54%, sys=1.95%, ctx=29155, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=29151/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=116604KB, aggrb=3886KB/s, minb=3886KB/s, maxb=3886KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdb: ios=29065/0, merge=0/0, ticks=29446/0, in_queue=29433, util=98.12%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.1% done] [0K/4K /s] [0 /1 iops] [eta 09h:47m:09s]
writetest: (groupid=0, jobs=1): err= 0: pid=3817
write: io=11136KB, bw=358153 B/s, iops=87, runt= 31839msec
slat (usec): min=3, max=596124, avg=10899.63, stdev=65581.18
clat (msec): min=1487, max=16067, avg=10136.40, stdev=2980.73
lat (msec): min=1954, max=16067, avg=10147.30, stdev=2981.04
clat percentiles (msec):
| 1.00th=[ 1958], 5.00th=[ 3392], 10.00th=[ 4948], 20.00th=[ 8356],
| 30.00th=[ 9896], 40.00th=[10421], 50.00th=[10814], 60.00th=[11207],
| 70.00th=[11731], 80.00th=[12387], 90.00th=[13304], 95.00th=[13698],
| 99.00th=[14353], 99.50th=[15008], 99.90th=[16057], 99.95th=[16057],
| 99.99th=[16057]
bw (KB/s): min= 0, max= 450, per=93.22%, avg=325.33, stdev=106.70
lat (msec): 2000=1.19%, >=2000=98.81%
cpu: usr=0.06%, sys=0.03%, ctx=211, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.3%, 16=0.6%, 32=1.1%, >=64=97.7%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=0/w=0/d=2784, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=11136KB, aggrb=349KB/s, minb=349KB/s, maxb=349KB/s, mint=31839msec, maxt=31839msec

Disk stats (read/write):
sdb: ios=84/2770, merge=0/0, ticks=70/4360043, in_queue=4409040, util=99.94%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=64
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.2% done] [0K/36K /s] [0 /9 iops] [eta 04h:40m:56s]
writetest: (groupid=0, jobs=1): err= 0: pid=3849
write: io=14500KB, bw=480082 B/s, iops=117, runt= 30928msec
slat (usec): min=3, max=32, avg=10.09, stdev= 2.61
clat (msec): min=1, max=2803, avg=545.79, stdev=512.00
lat (msec): min=1, max=2803, avg=545.80, stdev=512.00
clat percentiles (msec):
| 1.00th=[ 6], 5.00th=[ 12], 10.00th=[ 23], 20.00th=[ 120],
| 30.00th=[ 155], 40.00th=[ 251], 50.00th=[ 371], 60.00th=[ 553],
| 70.00th=[ 734], 80.00th=[ 963], 90.00th=[ 1319], 95.00th=[ 1565],
| 99.00th=[ 2024], 99.50th=[ 2180], 99.90th=[ 2507], 99.95th=[ 2606],
| 99.99th=[ 2802]
bw (KB/s): min= 81, max= 861, per=99.37%, avg=465.04, stdev=170.84
lat (msec): 2=0.03%, 4=0.03%, 10=4.39%, 20=4.77%, 50=5.63%
lat (msec): 100=2.34%, 250=22.79%, 500=17.38%, 750=13.46%, 1000=10.01%
lat (msec): 2000=18.15%, >=2000=1.02%
cpu: usr=0.32%, sys=0.41%, ctx=3626, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=0.9%, >=64=98.3%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued: total=r=0/w=0/d=3625, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=14500KB, aggrb=468KB/s, minb=468KB/s, maxb=468KB/s, mint=30928msec, maxt=30928msec

Disk stats (read/write):
sdb: ios=84/3619, merge=0/0, ticks=67/1952100, in_queue=1959030, util=99.94%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/308K /s] [0 /77 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3880
write: io=15960KB, bw=536911 B/s, iops=131, runt= 30439msec
slat (usec): min=3, max=29, avg=10.21, stdev= 2.29
clat (msec): min=1, max=1277, avg=244.01, stdev=222.90
lat (msec): min=1, max=1277, avg=244.02, stdev=222.90
clat percentiles (msec):
| 1.00th=[ 6], 5.00th=[ 10], 10.00th=[ 16], 20.00th=[ 33],
| 30.00th=[ 117], 40.00th=[ 141], 50.00th=[ 167], 60.00th=[ 241],
| 70.00th=[ 326], 80.00th=[ 408], 90.00th=[ 570], 95.00th=[ 685],
| 99.00th=[ 963], 99.50th=[ 1037], 99.90th=[ 1205], 99.95th=[ 1254],
| 99.99th=[ 1270]
bw (KB/s): min= 111, max= 814, per=99.65%, avg=522.18, stdev=146.44
lat (msec): 2=0.03%, 4=0.03%, 10=5.61%, 20=8.30%, 50=9.85%
lat (msec): 100=3.58%, 250=33.58%, 500=25.91%, 750=9.72%, 1000=2.63%
lat (msec): 2000=0.75%
cpu: usr=0.23%, sys=0.61%, ctx=3993, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=99.2%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3990, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=15960KB, aggrb=524KB/s, minb=524KB/s, maxb=524KB/s, mint=30439msec, maxt=30439msec

Disk stats (read/write):
sdb: ios=84/3987, merge=0/0, ticks=70/966153, in_queue=967613, util=99.95%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/292K /s] [0 /73 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3913
write: io=17028KB, bw=578004 B/s, iops=141, runt= 30167msec
slat (usec): min=3, max=33, avg= 9.80, stdev= 2.56
clat (msec): min=1, max=703, avg=113.33, stdev=112.35
lat (msec): min=1, max=703, avg=113.34, stdev=112.35
clat percentiles (msec):
| 1.00th=[ 6], 5.00th=[ 8], 10.00th=[ 10], 20.00th=[ 16],
| 30.00th=[ 24], 40.00th=[ 38], 50.00th=[ 109], 60.00th=[ 131],
| 70.00th=[ 143], 80.00th=[ 172], 90.00th=[ 277], 95.00th=[ 347],
| 99.00th=[ 469], 99.50th=[ 482], 99.90th=[ 611], 99.95th=[ 685],
| 99.99th=[ 701]
bw (KB/s): min= 224, max= 894, per=100.00%, avg=565.96, stdev=161.13
lat (msec): 2=0.02%, 4=0.02%, 10=11.28%, 20=14.02%, 50=20.39%
lat (msec): 100=3.24%, 250=39.53%, 500=11.18%, 750=0.31%
cpu: usr=0.16%, sys=0.69%, ctx=4259, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=99.6%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=4257, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=17028KB, aggrb=564KB/s, minb=564KB/s, maxb=564KB/s, mint=30167msec, maxt=30167msec

Disk stats (read/write):
sdb: ios=84/4255, merge=0/0, ticks=73/480767, in_queue=481510, util=99.96%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/328K /s] [0 /82 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3945
write: io=17428KB, bw=592348 B/s, iops=144, runt= 30128msec
slat (usec): min=3, max=60, avg=10.30, stdev= 2.35
clat (msec): min=1, max=439, avg=55.27, stdev=74.63
lat (msec): min=1, max=439, avg=55.28, stdev=74.63
clat percentiles (msec):
| 1.00th=[ 6], 5.00th=[ 6], 10.00th=[ 8], 20.00th=[ 10],
| 30.00th=[ 12], 40.00th=[ 15], 50.00th=[ 17], 60.00th=[ 21],
| 70.00th=[ 29], 80.00th=[ 121], 90.00th=[ 130], 95.00th=[ 212],
| 99.00th=[ 322], 99.50th=[ 326], 99.90th=[ 433], 99.95th=[ 433],
| 99.99th=[ 441]
bw (KB/s): min= 248, max= 918, per=100.00%, avg=583.55, stdev=168.80
lat (msec): 2=0.02%, 4=0.09%, 10=23.96%, 20=34.11%, 50=12.55%
lat (msec): 100=1.54%, 250=24.37%, 500=3.35%
cpu: usr=0.39%, sys=0.53%, ctx=4363, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=99.8%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=4357, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=17428KB, aggrb=578KB/s, minb=578KB/s, maxb=578KB/s, mint=30128msec, maxt=30128msec

Disk stats (read/write):
sdb: ios=84/4347, merge=0/0, ticks=70/238777, in_queue=239110, util=99.95%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/220K /s] [0 /55 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3977
write: io=17756KB, bw=605445 B/s, iops=147, runt= 30031msec
slat (usec): min=3, max=27, avg=10.13, stdev= 2.30
clat (msec): min=3, max=429, avg=27.05, stdev=51.71
lat (msec): min=3, max=429, avg=27.06, stdev=51.71
clat percentiles (msec):
| 1.00th=[ 6], 5.00th=[ 6], 10.00th=[ 6], 20.00th=[ 6],
| 30.00th=[ 8], 40.00th=[ 8], 50.00th=[ 8], 60.00th=[ 8],
| 70.00th=[ 10], 80.00th=[ 11], 90.00th=[ 113], 95.00th=[ 117],
| 99.00th=[ 233], 99.50th=[ 306], 99.90th=[ 314], 99.95th=[ 314],
| 99.99th=[ 429]
bw (KB/s): min= 160, max= 902, per=100.00%, avg=592.20, stdev=151.52
lat (msec): 4=0.14%, 10=79.41%, 20=5.54%, 100=1.04%, 250=12.93%
lat (msec): 500=0.95%
cpu: usr=0.26%, sys=0.68%, ctx=4445, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=4439, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=17756KB, aggrb=591KB/s, minb=591KB/s, maxb=591KB/s, mint=30031msec, maxt=30031msec

Disk stats (read/write):
sdb: ios=84/4432, merge=0/0, ticks=70/119270, in_queue=119460, util=99.95%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/316K /s] [0 /79 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=4008
write: io=17984KB, bw=613056 B/s, iops=149, runt= 30039msec
slat (usec): min=4, max=131, avg=12.93, stdev= 4.61
clat (msec): min=1, max=306, avg=13.34, stdev=35.30
lat (msec): min=2, max=306, avg=13.36, stdev=35.30
clat percentiles (msec):
| 1.00th=[ 4], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 4],
| 30.00th=[ 4], 40.00th=[ 4], 50.00th=[ 4], 60.00th=[ 4],
| 70.00th=[ 4], 80.00th=[ 4], 90.00th=[ 5], 95.00th=[ 109],
| 99.00th=[ 194], 99.50th=[ 200], 99.90th=[ 233], 99.95th=[ 302],
| 99.99th=[ 306]
bw (KB/s): min= 242, max= 862, per=100.00%, avg=599.47, stdev=132.12
lat (msec): 2=0.02%, 4=89.03%, 10=2.54%, 20=0.80%, 100=0.29%
lat (msec): 250=7.23%, 500=0.09%
cpu: usr=0.18%, sys=0.85%, ctx=4500, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=4496, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=17984KB, aggrb=598KB/s, minb=598KB/s, maxb=598KB/s, mint=30039msec, maxt=30039msec

Disk stats (read/write):
sdb: ios=84/4485, merge=0/0, ticks=74/59624, in_queue=59697, util=99.96%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/456K /s] [0 /114 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=4039
write: io=18128KB, bw=618686 B/s, iops=151, runt= 30004msec
slat (usec): min=5, max=42, avg=14.46, stdev= 3.39
clat (msec): min=1, max=293, avg= 6.60, stdev=24.74
lat (msec): min=1, max=293, avg= 6.62, stdev=24.74
clat percentiles (usec):
| 1.00th=[ 1720], 5.00th=[ 1752], 10.00th=[ 1800], 20.00th=[ 1832],
| 30.00th=[ 1848], 40.00th=[ 1864], 50.00th=[ 1912], 60.00th=[ 1944],
| 70.00th=[ 1976], 80.00th=[ 1992], 90.00th=[ 2024], 95.00th=[ 2448],
| 99.00th=[109056], 99.50th=[187392], 99.90th=[197632], 99.95th=[228352],
| 99.99th=[292864]
bw (KB/s): min= 207, max= 896, per=100.00%, avg=607.58, stdev=153.16
lat (msec): 2=83.94%, 4=11.50%, 10=0.84%, 20=0.02%, 100=0.13%
lat (msec): 250=3.55%, 500=0.02%
cpu: usr=0.13%, sys=0.91%, ctx=4540, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=4532, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=18128KB, aggrb=604KB/s, minb=604KB/s, maxb=604KB/s, mint=30004msec, maxt=30004msec

Disk stats (read/write):
sdb: ios=84/4524, merge=0/0, ticks=70/29663, in_queue=29770, util=99.24%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [5248K/0K /s] [1312 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4616
read: io=157824KB, bw=5240.2KB/s, iops=1310, runt= 30118msec
slat (usec): min=2, max=25379, avg=758.25, stdev=4208.51
clat (msec): min=98, max=1956, avg=773.34, stdev=139.05
lat (msec): min=122, max=1956, avg=774.10, stdev=139.16
clat percentiles (msec):
| 1.00th=[ 416], 5.00th=[ 685], 10.00th=[ 685], 20.00th=[ 709],
| 30.00th=[ 709], 40.00th=[ 734], 50.00th=[ 734], 60.00th=[ 758],
| 70.00th=[ 783], 80.00th=[ 807], 90.00th=[ 906], 95.00th=[ 1057],
| 99.00th=[ 1319], 99.50th=[ 1434], 99.90th=[ 1598], 99.95th=[ 1663],
| 99.99th=[ 1811]
bw (KB/s): min= 5, max= 5280, per=98.38%, avg=5155.14, stdev=688.17
lat (msec): 100=0.01%, 250=0.49%, 500=0.81%, 750=51.20%, 1000=40.98%
lat (msec): 2000=6.53%
cpu: usr=0.33%, sys=0.66%, ctx=1365, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=39456/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=157824KB, aggrb=5240KB/s, minb=5240KB/s, maxb=5240KB/s, mint=30118msec, maxt=30118msec

Disk stats (read/write):
sdb: ios=39187/0, merge=0/0, ticks=4274397/0, in_queue=4285950, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [5112K/0K /s] [1278 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4622
read: io=153360KB, bw=5107.6KB/s, iops=1276, runt= 30026msec
slat (usec): min=3, max=118, avg= 4.74, stdev= 2.30
clat (usec): min=900, max=66226, avg=25052.12, stdev=13451.25
lat (usec): min=936, max=66230, avg=25057.28, stdev=13451.24
clat percentiles (usec):
| 1.00th=[ 2384], 5.00th=[ 4016], 10.00th=[ 6368], 20.00th=[11072],
| 30.00th=[15808], 40.00th=[20352], 50.00th=[24960], 60.00th=[29568],
| 70.00th=[34048], 80.00th=[38656], 90.00th=[43264], 95.00th=[46336],
| 99.00th=[50944], 99.50th=[52480], 99.90th=[55552], 99.95th=[56064],
| 99.99th=[60672]
bw (KB/s): min= 4864, max= 5120, per=100.00%, avg=5108.00, stdev=32.89
lat (usec): 1000=0.01%
lat (msec): 2=0.01%, 4=4.91%, 10=12.15%, 20=22.02%, 50=59.51%
lat (msec): 100=1.40%
cpu: usr=0.73%, sys=1.43%, ctx=38345, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=38340/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=153360KB, aggrb=5107KB/s, minb=5107KB/s, maxb=5107KB/s, mint=30026msec, maxt=30026msec

Disk stats (read/write):
sdb: ios=38192/0, merge=1/0, ticks=956237/0, in_queue=956730, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4944K/0K /s] [1236 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4626
read: io=148328KB, bw=4941.1KB/s, iops=1235, runt= 30014msec
slat (usec): min=3, max=118, avg= 4.83, stdev= 2.38
clat (usec): min=905, max=37620, avg=12942.28, stdev=6474.59
lat (usec): min=940, max=37624, avg=12947.54, stdev=6474.59
clat percentiles (usec):
| 1.00th=[ 2352], 5.00th=[ 3152], 10.00th=[ 4128], 20.00th=[ 6368],
| 30.00th=[ 8640], 40.00th=[10560], 50.00th=[12864], 60.00th=[14912],
| 70.00th=[17024], 80.00th=[19328], 90.00th=[21632], 95.00th=[23424],
| 99.00th=[26240], 99.50th=[27264], 99.90th=[29568], 99.95th=[30080],
| 99.99th=[31872]
bw (KB/s): min= 4920, max= 4976, per=100.00%, avg=4946.44, stdev=10.84
lat (usec): 1000=0.01%
lat (msec): 2=0.01%, 4=9.13%, 10=28.24%, 20=45.49%, 50=17.14%
cpu: usr=0.66%, sys=1.63%, ctx=37089, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=37082/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=148328KB, aggrb=4941KB/s, minb=4941KB/s, maxb=4941KB/s, mint=30014msec, maxt=30014msec

Disk stats (read/write):
sdb: ios=36956/0, merge=0/0, ticks=478014/0, in_queue=478134, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4756K/0K /s] [1189 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4635
read: io=142896KB, bw=4762.9KB/s, iops=1190, runt= 30007msec
slat (usec): min=3, max=122, avg= 5.18, stdev= 2.62
clat (usec): min=895, max=18630, avg=6711.70, stdev=2806.36
lat (usec): min=932, max=18635, avg=6717.32, stdev=2806.35
clat percentiles (usec):
| 1.00th=[ 2384], 5.00th=[ 2480], 10.00th=[ 3216], 20.00th=[ 4080],
| 30.00th=[ 4960], 40.00th=[ 5792], 50.00th=[ 6624], 60.00th=[ 7520],
| 70.00th=[ 8384], 80.00th=[ 9280], 90.00th=[10432], 95.00th=[11584],
| 99.00th=[13248], 99.50th=[14016], 99.90th=[15040], 99.95th=[15680],
| 99.99th=[17536]
bw (KB/s): min= 4744, max= 4808, per=100.00%, avg=4766.51, stdev=12.91
lat (usec): 1000=0.01%
lat (msec): 2=0.01%, 4=19.26%, 10=66.98%, 20=13.76%
cpu: usr=0.84%, sys=1.30%, ctx=35731, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=35724/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=142896KB, aggrb=4762KB/s, minb=4762KB/s, maxb=4762KB/s, mint=30007msec, maxt=30007msec

Disk stats (read/write):
sdb: ios=35613/0, merge=0/0, ticks=238850/0, in_queue=238857, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4628K/0K /s] [1157 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4640
read: io=138816KB, bw=4626.6KB/s, iops=1156, runt= 30004msec
slat (usec): min=3, max=111, avg= 5.17, stdev= 2.53
clat (usec): min=888, max=8495, avg=3450.49, stdev=810.23
lat (usec): min=924, max=8499, avg=3456.10, stdev=810.16
clat percentiles (usec):
| 1.00th=[ 2448], 5.00th=[ 2480], 10.00th=[ 2576], 20.00th=[ 2608],
| 30.00th=[ 2640], 40.00th=[ 3376], 50.00th=[ 3472], 60.00th=[ 3504],
| 70.00th=[ 3504], 80.00th=[ 4256], 90.00th=[ 4384], 95.00th=[ 5152],
| 99.00th=[ 5984], 99.50th=[ 6112], 99.90th=[ 6880], 99.95th=[ 7520],
| 99.99th=[ 7776]
bw (KB/s): min= 4616, max= 4656, per=100.00%, avg=4630.92, stdev= 8.59
lat (usec): 1000=0.01%
lat (msec): 2=0.01%, 4=74.99%, 10=25.00%
cpu: usr=0.60%, sys=1.50%, ctx=34710, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=34704/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=138816KB, aggrb=4626KB/s, minb=4626KB/s, maxb=4626KB/s, mint=30004msec, maxt=30004msec

Disk stats (read/write):
sdb: ios=34598/0, merge=0/0, ticks=119153/0, in_queue=119123, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4608K/0K /s] [1152 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4646
read: io=138268KB, bw=4608.8KB/s, iops=1152, runt= 30001msec
slat (usec): min=3, max=125, avg= 6.25, stdev= 3.06
clat (usec): min=884, max=2286, avg=1727.08, stdev=53.30
lat (usec): min=920, max=2291, avg=1733.79, stdev=52.82
clat percentiles (usec):
| 1.00th=[ 1576], 5.00th=[ 1608], 10.00th=[ 1624], 20.00th=[ 1720],
| 30.00th=[ 1720], 40.00th=[ 1736], 50.00th=[ 1736], 60.00th=[ 1752],
| 70.00th=[ 1752], 80.00th=[ 1752], 90.00th=[ 1768], 95.00th=[ 1768],
| 99.00th=[ 1864], 99.50th=[ 1864], 99.90th=[ 2008], 99.95th=[ 2224],
| 99.99th=[ 2256]
bw (KB/s): min= 4600, max= 4624, per=100.00%, avg=4612.75, stdev= 8.15
lat (usec): 1000=0.01%
lat (msec): 2=99.90%, 4=0.10%
cpu: usr=0.61%, sys=1.60%, ctx=34578, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=34567/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=138268KB, aggrb=4608KB/s, minb=4608KB/s, maxb=4608KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdb: ios=34465/0, merge=0/0, ticks=59313/0, in_queue=59287, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [4592K/0K /s] [1148 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4650
read: io=137812KB, bw=4593.6KB/s, iops=1148, runt= 30001msec
slat (usec): min=4, max=119, avg= 7.87, stdev= 3.64
clat (usec): min=688, max=6641, avg=860.36, stdev=54.23
lat (usec): min=704, max=6655, avg=868.71, stdev=54.21
clat percentiles (usec):
| 1.00th=[ 724], 5.00th=[ 748], 10.00th=[ 836], 20.00th=[ 852],
| 30.00th=[ 852], 40.00th=[ 868], 50.00th=[ 868], 60.00th=[ 868],
| 70.00th=[ 868], 80.00th=[ 884], 90.00th=[ 884], 95.00th=[ 900],
| 99.00th=[ 980], 99.50th=[ 988], 99.90th=[ 1096], 99.95th=[ 1112],
| 99.99th=[ 1144]
bw (KB/s): min= 4568, max= 4624, per=100.00%, avg=4597.69, stdev=12.16
lat (usec): 750=4.97%, 1000=94.82%
lat (msec): 2=0.20%, 10=0.01%
cpu: usr=0.69%, sys=1.75%, ctx=34459, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=34453/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=137812KB, aggrb=4593KB/s, minb=4593KB/s, maxb=4593KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdb: ios=34352/0, merge=0/0, ticks=29370/0, in_queue=29347, util=97.83%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/0K /s] [0 /0 iops] [eta 107d:03h:32m:23s]
writetest: (groupid=0, jobs=1): err= 0: pid=11326
write: io=307200 B, bw=5960 B/s, iops=1, runt= 51535msec
slat (usec): min=3, max=26, avg=11.08, stdev= 6.44
clat (msec): min=684, max=49451, avg=21703.81, stdev=13250.41
lat (msec): min=684, max=49451, avg=21703.82, stdev=13250.41
clat percentiles (msec):
| 1.00th=[ 685], 5.00th=[ 2057], 10.00th=[ 4146], 20.00th=[ 8225],
| 30.00th=[11731], 40.00th=[16712], 50.00th=[16712], 60.00th=[16712],
| 70.00th=[16712], 80.00th=[16712], 90.00th=[16712], 95.00th=[16712],
| 99.00th=[16712], 99.50th=[16712], 99.90th=[16712], 99.95th=[16712],
| 99.99th=[16712]
bw (KB/s): min= 0, max= 5, per=97.78%, avg= 4.89, stdev= 0.75
lat (msec): 750=1.33%, 2000=2.67%, >=2000=96.00%
cpu: usr=0.01%, sys=0.00%, ctx=76, majf=0, minf=0
IO depths: 1=1.3%, 2=2.7%, 4=5.3%, 8=10.7%, 16=21.3%, 32=58.7%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=97.8%, 8=0.0%, 16=0.0%, 32=2.2%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=75, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=300KB, aggrb=5KB/s, minb=5KB/s, maxb=5KB/s, mint=51535msec, maxt=51535msec

Disk stats (read/write):
sdb: ios=84/74, merge=0/0, ticks=70/1283090, in_queue=1308454, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/0K /s] [0 /0 iops] [eta 84d:00h:53m:38s]
writetest: (groupid=0, jobs=1): err= 0: pid=18036
write: io=241664 B, bw=5956 B/s, iops=1, runt= 40571msec
slat (usec): min=3, max=25, avg=12.86, stdev= 5.71
clat (msec): min=684, max=24058, avg=10826.78, stdev=5773.72
lat (msec): min=684, max=24058, avg=10826.79, stdev=5773.72
clat percentiles (msec):
| 1.00th=[ 685], 5.00th=[ 1401], 10.00th=[ 2089], 20.00th=[ 4817],
| 30.00th=[ 6915], 40.00th=[ 9634], 50.00th=[11731], 60.00th=[13042],
| 70.00th=[14484], 80.00th=[15139], 90.00th=[16712], 95.00th=[16712],
| 99.00th=[16712], 99.50th=[16712], 99.90th=[16712], 99.95th=[16712],
| 99.99th=[16712]
bw (KB/s): min= 0, max= 5, per=97.78%, avg= 4.89, stdev= 0.75
lat (msec): 750=1.69%, 2000=3.39%, >=2000=94.92%
cpu: usr=0.01%, sys=0.00%, ctx=60, majf=0, minf=0
IO depths: 1=1.7%, 2=3.4%, 4=6.8%, 8=13.6%, 16=74.6%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=97.8%, 8=0.0%, 16=2.2%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=59, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=236KB, aggrb=5KB/s, minb=5KB/s, maxb=5KB/s, mint=40571msec, maxt=40571msec

Disk stats (read/write):
sdb: ios=84/58, merge=0/0, ticks=67/552507, in_queue=566187, util=99.97%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/0K /s] [0 /0 iops] [eta 73d:12h:46m:55s]
writetest: (groupid=0, jobs=1): err= 0: pid=22634
write: io=208896 B, bw=5958 B/s, iops=1, runt= 35061msec
slat (usec): min=3, max=27, avg=15.22, stdev= 4.75
clat (msec): min=685, max=10998, avg=5404.96, stdev=2526.07
lat (msec): min=685, max=10998, avg=5404.97, stdev=2526.07
clat percentiles (msec):
| 1.00th=[ 685], 5.00th=[ 1369], 10.00th=[ 1401], 20.00th=[ 2737],
| 30.00th=[ 4113], 40.00th=[ 4817], 50.00th=[ 6194], 60.00th=[ 6849],
| 70.00th=[ 6915], 80.00th=[ 7570], 90.00th=[ 8225], 95.00th=[ 8225],
| 99.00th=[10945], 99.50th=[10945], 99.90th=[10945], 99.95th=[10945],
| 99.99th=[10945]
bw (KB/s): min= 0, max= 5, per=97.78%, avg= 4.89, stdev= 0.75
lat (msec): 750=1.96%, 2000=9.80%, >=2000=88.24%
cpu: usr=0.00%, sys=0.01%, ctx=52, majf=0, minf=0
IO depths: 1=2.0%, 2=3.9%, 4=7.8%, 8=86.3%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=97.8%, 8=2.2%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=51, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=204KB, aggrb=5KB/s, minb=5KB/s, maxb=5KB/s, mint=35061msec, maxt=35061msec

Disk stats (read/write):
sdb: ios=84/50, merge=0/0, ticks=70/255730, in_queue=261170, util=99.97%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/0K /s] [0 /0 iops] [eta 67d:05h:30m:54s]
writetest: (groupid=0, jobs=1): err= 0: pid=26634
write: io=192512 B, bw=5956 B/s, iops=1, runt= 32322msec
slat (usec): min=3, max=25, avg=15.77, stdev= 3.33
clat (msec): min=684, max=4825, avg=2706.33, stdev=852.43
lat (msec): min=684, max=4825, avg=2706.34, stdev=852.43
clat percentiles (msec):
| 1.00th=[ 685], 5.00th=[ 1369], 10.00th=[ 1385], 20.00th=[ 2057],
| 30.00th=[ 2089], 40.00th=[ 2737], 50.00th=[ 2737], 60.00th=[ 2737],
| 70.00th=[ 3425], 80.00th=[ 3425], 90.00th=[ 3458], 95.00th=[ 4113],
| 99.00th=[ 4817], 99.50th=[ 4817], 99.90th=[ 4817], 99.95th=[ 4817],
| 99.99th=[ 4817]
bw (KB/s): min= 1, max= 5, per=98.22%, avg= 4.91, stdev= 0.60
lat (msec): 750=2.13%, 2000=10.64%, >=2000=87.23%
cpu: usr=0.01%, sys=0.00%, ctx=48, majf=0, minf=0
IO depths: 1=2.1%, 2=4.3%, 4=93.6%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=47, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=188KB, aggrb=5KB/s, minb=5KB/s, maxb=5KB/s, mint=32322msec, maxt=32322msec

Disk stats (read/write):
sdb: ios=84/46, merge=0/0, ticks=67/121626, in_queue=124990, util=99.94%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/4K /s] [0 /1 iops] [eta 65d:03h:05m:34s]
writetest: (groupid=0, jobs=1): err= 0: pid=30700
write: io=184320 B, bw=5961 B/s, iops=1, runt= 30919msec
slat (usec): min=5, max=25, avg=16.40, stdev= 2.69
clat (msec): min=686, max=1400, avg=1358.91, stdev=102.85
lat (msec): min=686, max=1400, avg=1358.93, stdev=102.85
clat percentiles (msec):
| 1.00th=[ 685], 5.00th=[ 1369], 10.00th=[ 1369], 20.00th=[ 1369],
| 30.00th=[ 1369], 40.00th=[ 1369], 50.00th=[ 1369], 60.00th=[ 1369],
| 70.00th=[ 1369], 80.00th=[ 1369], 90.00th=[ 1385], 95.00th=[ 1401],
| 99.00th=[ 1401], 99.50th=[ 1401], 99.90th=[ 1401], 99.95th=[ 1401],
| 99.99th=[ 1401]
bw (KB/s): min= 5, max= 5, per=100.00%, avg= 5.00, stdev= 0.00
lat (msec): 750=2.22%, 2000=97.78%
cpu: usr=0.00%, sys=0.01%, ctx=46, majf=0, minf=0
IO depths: 1=2.2%, 2=97.8%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=45, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=180KB, aggrb=5KB/s, minb=5KB/s, maxb=5KB/s, mint=30919msec, maxt=30919msec

Disk stats (read/write):
sdb: ios=84/44, merge=0/0, ticks=66/59776, in_queue=61210, util=99.94%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/4K /s] [0 /1 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=1796
write: io=180224 B, bw=5960 B/s, iops=1, runt= 30237msec
slat (usec): min=19, max=26, avg=20.86, stdev= 1.44
clat (msec): min=684, max=715, avg=687.15, stdev= 6.12
lat (msec): min=684, max=715, avg=687.17, stdev= 6.12
clat percentiles (msec):
| 1.00th=[ 685], 5.00th=[ 685], 10.00th=[ 685], 20.00th=[ 685],
| 30.00th=[ 685], 40.00th=[ 685], 50.00th=[ 685], 60.00th=[ 685],
| 70.00th=[ 685], 80.00th=[ 685], 90.00th=[ 685], 95.00th=[ 693],
| 99.00th=[ 717], 99.50th=[ 717], 99.90th=[ 717], 99.95th=[ 717],
| 99.99th=[ 717]
bw (KB/s): min= 5, max= 5, per=100.00%, avg= 5.00, stdev= 0.00
lat (msec): 750=100.00%
cpu: usr=0.01%, sys=0.00%, ctx=45, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=44, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=176KB, aggrb=5KB/s, minb=5KB/s, maxb=5KB/s, mint=30237msec, maxt=30237msec

Disk stats (read/write):
sdb: ios=84/43, merge=0/0, ticks=67/29550, in_queue=30230, util=99.94%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [5.6% done] [3204K/0K /s] [801 /0 iops] [eta 08m:38s]
readtest: (groupid=0, jobs=1): err= 0: pid=4182
read: io=114640KB, bw=3800.9KB/s, iops=950, runt= 30162msec
slat (usec): min=2, max=68303, avg=1045.05, stdev=5824.60
clat (msec): min=136, max=1823, avg=1059.66, stdev=161.31
lat (msec): min=170, max=1823, avg=1060.71, stdev=161.40
clat percentiles (msec):
| 1.00th=[ 437], 5.00th=[ 938], 10.00th=[ 938], 20.00th=[ 971],
| 30.00th=[ 979], 40.00th=[ 1004], 50.00th=[ 1020], 60.00th=[ 1074],
| 70.00th=[ 1106], 80.00th=[ 1172], 90.00th=[ 1270], 95.00th=[ 1352],
| 99.00th=[ 1516], 99.50th=[ 1549], 99.90th=[ 1647], 99.95th=[ 1680],
| 99.99th=[ 1795]
bw (KB/s): min= 3, max= 4008, per=98.71%, avg=3750.79, stdev=502.34
lat (msec): 250=0.34%, 500=0.78%, 750=0.89%, 1000=34.54%, 2000=63.45%
cpu: usr=0.23%, sys=0.46%, ctx=1029, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=28660/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=114640KB, aggrb=3800KB/s, minb=3800KB/s, maxb=3800KB/s, mint=30162msec, maxt=30162msec

Disk stats (read/write):
sdb: ios=28639/0, merge=20/0, ticks=4312340/0, in_queue=4312670, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [3608K/0K /s] [902 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4187
read: io=108436KB, bw=3610.3KB/s, iops=902, runt= 30036msec
slat (usec): min=3, max=122, avg= 5.34, stdev= 2.71
clat (msec): min=2, max=94, avg=35.45, stdev=19.17
lat (msec): min=2, max=94, avg=35.45, stdev=19.17
clat percentiles (usec):
| 1.00th=[ 3024], 5.00th=[ 5600], 10.00th=[ 9024], 20.00th=[15680],
| 30.00th=[22144], 40.00th=[28800], 50.00th=[35584], 60.00th=[41728],
| 70.00th=[48384], 80.00th=[54528], 90.00th=[61696], 95.00th=[65280],
| 99.00th=[72192], 99.50th=[74240], 99.90th=[78336], 99.95th=[79360],
| 99.99th=[84480]
bw (KB/s): min= 3584, max= 3640, per=100.00%, avg=3614.12, stdev=14.16
lat (msec): 4=2.71%, 10=8.99%, 20=14.90%, 50=45.97%, 100=27.43%
cpu: usr=0.61%, sys=1.22%, ctx=27123, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=99.9%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=27109/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=108436KB, aggrb=3610KB/s, minb=3610KB/s, maxb=3610KB/s, mint=30036msec, maxt=30036msec

Disk stats (read/write):
sdb: ios=26993/0, merge=3/0, ticks=955993/0, in_queue=956693, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [3268K/0K /s] [817 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4193
read: io=98564KB, bw=3283.4KB/s, iops=820, runt= 30019msec
slat (usec): min=3, max=121, avg= 5.26, stdev= 2.57
clat (msec): min=2, max=50, avg=19.48, stdev= 9.76
lat (msec): min=2, max=50, avg=19.49, stdev= 9.76
clat percentiles (usec):
| 1.00th=[ 3024], 5.00th=[ 4576], 10.00th=[ 6112], 20.00th=[ 9536],
| 30.00th=[12736], 40.00th=[16064], 50.00th=[19328], 60.00th=[22656],
| 70.00th=[25728], 80.00th=[29056], 90.00th=[32640], 95.00th=[35072],
| 99.00th=[39168], 99.50th=[40704], 99.90th=[43776], 99.95th=[44800],
| 99.99th=[45824]
bw (KB/s): min= 3224, max= 3328, per=100.00%, avg=3286.44, stdev=21.13
lat (msec): 4=4.12%, 10=17.36%, 20=30.77%, 50=47.74%, 100=0.01%
cpu: usr=0.72%, sys=1.05%, ctx=24650, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=99.9%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=24641/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=98564KB, aggrb=3283KB/s, minb=3283KB/s, maxb=3283KB/s, mint=30019msec, maxt=30019msec

Disk stats (read/write):
sdb: ios=24552/0, merge=0/0, ticks=477986/0, in_queue=478184, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [2828K/0K /s] [707 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4198
read: io=85132KB, bw=2836.8KB/s, iops=709, runt= 30011msec
slat (usec): min=3, max=124, avg= 5.84, stdev= 2.78
clat (msec): min=1, max=30, avg=11.27, stdev= 4.81
lat (msec): min=2, max=30, avg=11.28, stdev= 4.81
clat percentiles (usec):
| 1.00th=[ 3536], 5.00th=[ 4048], 10.00th=[ 4832], 20.00th=[ 6432],
| 30.00th=[ 8032], 40.00th=[ 9536], 50.00th=[11072], 60.00th=[12608],
| 70.00th=[14144], 80.00th=[15808], 90.00th=[17792], 95.00th=[19328],
| 99.00th=[22144], 99.50th=[23168], 99.90th=[25216], 99.95th=[26240],
| 99.99th=[27520]
bw (KB/s): min= 2752, max= 2920, per=100.00%, avg=2839.36, stdev=32.02
lat (msec): 2=0.01%, 4=4.87%, 10=38.22%, 20=53.46%, 50=3.44%
cpu: usr=0.78%, sys=1.11%, ctx=21293, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=21283/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=85132KB, aggrb=2836KB/s, minb=2836KB/s, maxb=2836KB/s, mint=30011msec, maxt=30011msec

Disk stats (read/write):
sdb: ios=21212/0, merge=0/0, ticks=238853/0, in_queue=238904, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [2440K/0K /s] [610 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4205
read: io=73440KB, bw=2447.4KB/s, iops=611, runt= 30008msec
slat (usec): min=3, max=133, avg= 6.30, stdev= 3.41
clat (msec): min=1, max=15, avg= 6.53, stdev= 1.66
lat (msec): min=1, max=15, avg= 6.53, stdev= 1.66
clat percentiles (usec):
| 1.00th=[ 3632], 5.00th=[ 4256], 10.00th=[ 4512], 20.00th=[ 5024],
| 30.00th=[ 5408], 40.00th=[ 5984], 50.00th=[ 6432], 60.00th=[ 6816],
| 70.00th=[ 7264], 80.00th=[ 7840], 90.00th=[ 8768], 95.00th=[ 9536],
| 99.00th=[11200], 99.50th=[11840], 99.90th=[12992], 99.95th=[13632],
| 99.99th=[15168]
bw (KB/s): min= 2408, max= 2530, per=100.00%, avg=2450.12, stdev=23.74
lat (msec): 2=0.01%, 4=2.61%, 10=94.20%, 20=3.18%
cpu: usr=0.93%, sys=0.84%, ctx=18373, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=18360/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=73440KB, aggrb=2447KB/s, minb=2447KB/s, maxb=2447KB/s, mint=30008msec, maxt=30008msec

Disk stats (read/write):
sdb: ios=18300/0, merge=0/0, ticks=119210/0, in_queue=119227, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [2416K/0K /s] [604 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4209
read: io=71640KB, bw=2387.9KB/s, iops=596, runt= 30002msec
slat (usec): min=3, max=35, avg= 9.64, stdev= 3.52
clat (usec): min=1185, max=5737, avg=3337.08, stdev=456.90
lat (usec): min=1213, max=5749, avg=3347.25, stdev=456.89
clat percentiles (usec):
| 1.00th=[ 1992], 5.00th=[ 2608], 10.00th=[ 2640], 20.00th=[ 2928],
| 30.00th=[ 3184], 40.00th=[ 3280], 50.00th=[ 3440], 60.00th=[ 3504],
| 70.00th=[ 3568], 80.00th=[ 3664], 90.00th=[ 3824], 95.00th=[ 3920],
| 99.00th=[ 4704], 99.50th=[ 4832], 99.90th=[ 4896], 99.95th=[ 4960],
| 99.99th=[ 5728]
bw (KB/s): min= 2352, max= 2432, per=100.00%, avg=2389.15, stdev=20.15
lat (msec): 2=1.14%, 4=96.04%, 10=2.82%
cpu: usr=0.73%, sys=1.76%, ctx=17923, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=17910/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=71640KB, aggrb=2387KB/s, minb=2387KB/s, maxb=2387KB/s, mint=30002msec, maxt=30002msec

Disk stats (read/write):
sdb: ios=17854/0, merge=0/0, ticks=59277/0, in_queue=59274, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [2364K/0K /s] [591 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4216
read: io=70868KB, bw=2362.2KB/s, iops=590, runt= 30001msec
slat (usec): min=4, max=125, avg=10.06, stdev= 4.19
clat (usec): min=929, max=3056, avg=1680.19, stdev=318.18
lat (usec): min=942, max=3071, avg=1690.77, stdev=318.27
clat percentiles (usec):
| 1.00th=[ 972], 5.00th=[ 988], 10.00th=[ 1096], 20.00th=[ 1592],
| 30.00th=[ 1608], 40.00th=[ 1624], 50.00th=[ 1640], 60.00th=[ 1848],
| 70.00th=[ 1864], 80.00th=[ 1880], 90.00th=[ 1976], 95.00th=[ 1992],
| 99.00th=[ 2864], 99.50th=[ 2864], 99.90th=[ 2992], 99.95th=[ 2992],
| 99.99th=[ 3024]
bw (KB/s): min= 2320, max= 2408, per=100.00%, avg=2364.36, stdev=22.11
lat (usec): 1000=8.00%
lat (msec): 2=88.82%, 4=3.18%
cpu: usr=0.58%, sys=1.23%, ctx=17729, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=17717/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=70868KB, aggrb=2362KB/s, minb=2362KB/s, maxb=2362KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdb: ios=17664/0, merge=0/0, ticks=29536/0, in_queue=29517, util=98.39%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/0K /s] [0 /0 iops] [eta 49710d:06h:27m:29s]
writetest: (groupid=0, jobs=1): err= 0: pid=4234
write: io=1660.0KB, bw=36269 B/s, iops=8, runt= 46867msec
slat (usec): min=3, max=4088.5K, avg=77160.94, stdev=520319.60
clat (msec): min=14843, max=46865, avg=37126.84, stdev=10034.11
lat (msec): min=18784, max=46865, avg=37204.00, stdev=9985.34
clat percentiles (msec):
| 1.00th=[16712], 5.00th=[16712], 10.00th=[16712], 20.00th=[16712],
| 30.00th=[16712], 40.00th=[16712], 50.00th=[16712], 60.00th=[16712],
| 70.00th=[16712], 80.00th=[16712], 90.00th=[16712], 95.00th=[16712],
| 99.00th=[16712], 99.50th=[16712], 99.90th=[16712], 99.95th=[16712],
| 99.99th=[16712]
bw (KB/s): min= 0, max= 0, per=0.00%, avg= 0.00, stdev= 0.00
lat (msec): >=2000=100.00%
cpu: usr=0.00%, sys=0.03%, ctx=138, majf=0, minf=0
IO depths: 1=0.2%, 2=0.5%, 4=1.0%, 8=1.9%, 16=3.9%, 32=7.7%, >=64=84.8%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=100.0%
issued: total=r=0/w=0/d=415, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1660KB, aggrb=35KB/s, minb=35KB/s, maxb=35KB/s, mint=46867msec, maxt=46867msec

Disk stats (read/write):
sdb: ios=84/413, merge=3/0, ticks=694/5472010, in_queue=5509883, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.1% done] [0K/0K /s] [0 /0 iops] [eta 17h:24m:15s]
writetest: (groupid=0, jobs=1): err= 0: pid=4264
write: io=1156.0KB, bw=35509 B/s, iops=8, runt= 33336msec
slat (usec): min=3, max=24, avg=11.15, stdev= 3.16
clat (msec): min=116, max=8382, avg=3679.60, stdev=1873.21
lat (msec): min=116, max=8382, avg=3679.62, stdev=1873.21
clat percentiles (msec):
| 1.00th=[ 255], 5.00th=[ 660], 10.00th=[ 1004], 20.00th=[ 1647],
| 30.00th=[ 2409], 40.00th=[ 3228], 50.00th=[ 3916], 60.00th=[ 4686],
| 70.00th=[ 5014], 80.00th=[ 5342], 90.00th=[ 5669], 95.00th=[ 6128],
| 99.00th=[ 7767], 99.50th=[ 7963], 99.90th=[ 8356], 99.95th=[ 8356],
| 99.99th=[ 8356]
bw (KB/s): min= 3, max= 53, per=98.61%, avg=33.53, stdev= 8.02
lat (msec): 250=0.69%, 500=3.11%, 750=2.42%, 1000=3.46%, 2000=14.88%
lat (msec): >=2000=75.43%
cpu: usr=0.00%, sys=0.06%, ctx=292, majf=0, minf=0
IO depths: 1=0.3%, 2=0.7%, 4=1.4%, 8=2.8%, 16=5.5%, 32=89.3%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=99.6%, 8=0.0%, 16=0.0%, 32=0.4%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=289, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1156KB, aggrb=34KB/s, minb=34KB/s, maxb=34KB/s, mint=33336msec, maxt=33336msec

Disk stats (read/write):
sdb: ios=84/287, merge=3/0, ticks=114/1004880, in_queue=1013170, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.1% done] [0K/4K /s] [0 /1 iops] [eta 16h:24m:47s]
writetest: (groupid=0, jobs=1): err= 0: pid=4295
write: io=1088.0KB, bw=35257 B/s, iops=8, runt= 31599msec
slat (usec): min=3, max=23, avg=11.47, stdev= 2.19
clat (msec): min=257, max=3878, avg=1852.79, stdev=850.88
lat (msec): min=257, max=3878, avg=1852.81, stdev=850.88
clat percentiles (msec):
| 1.00th=[ 293], 5.00th=[ 498], 10.00th=[ 619], 20.00th=[ 938],
| 30.00th=[ 1336], 40.00th=[ 1598], 50.00th=[ 1958], 60.00th=[ 2311],
| 70.00th=[ 2474], 80.00th=[ 2606], 90.00th=[ 2868], 95.00th=[ 3032],
| 99.00th=[ 3720], 99.50th=[ 3785], 99.90th=[ 3884], 99.95th=[ 3884],
| 99.99th=[ 3884]
bw (KB/s): min= 7, max= 49, per=98.50%, avg=33.49, stdev= 7.39
lat (msec): 500=5.15%, 750=9.93%, 1000=5.51%, 2000=29.78%, >=2000=49.63%
cpu: usr=0.00%, sys=0.06%, ctx=274, majf=0, minf=0
IO depths: 1=0.4%, 2=0.7%, 4=1.5%, 8=2.9%, 16=94.5%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=99.6%, 8=0.0%, 16=0.4%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=272, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1088KB, aggrb=34KB/s, minb=34KB/s, maxb=34KB/s, mint=31599msec, maxt=31599msec

Disk stats (read/write):
sdb: ios=84/270, merge=3/0, ticks=107/486823, in_queue=490777, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/36K /s] [0 /9 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=4327
write: io=1064.0KB, bw=35293 B/s, iops=8, runt= 30871msec
slat (usec): min=4, max=18, avg=11.48, stdev= 1.40
clat (msec): min=214, max=2006, avg=925.17, stdev=385.05
lat (msec): min=214, max=2006, avg=925.18, stdev=385.05
clat percentiles (msec):
| 1.00th=[ 245], 5.00th=[ 281], 10.00th=[ 400], 20.00th=[ 519],
| 30.00th=[ 652], 40.00th=[ 848], 50.00th=[ 971], 60.00th=[ 1074],
| 70.00th=[ 1205], 80.00th=[ 1303], 90.00th=[ 1385], 95.00th=[ 1483],
| 99.00th=[ 1729], 99.50th=[ 1926], 99.90th=[ 2008], 99.95th=[ 2008],
| 99.99th=[ 2008]
bw (KB/s): min= 13, max= 47, per=99.46%, avg=33.81, stdev= 6.65
lat (msec): 250=1.13%, 500=18.05%, 750=15.79%, 1000=18.05%, 2000=46.62%
lat (msec): >=2000=0.38%
cpu: usr=0.04%, sys=0.02%, ctx=269, majf=0, minf=0
IO depths: 1=0.4%, 2=0.8%, 4=1.5%, 8=97.4%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=99.6%, 8=0.4%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=266, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1064KB, aggrb=34KB/s, minb=34KB/s, maxb=34KB/s, mint=30871msec, maxt=30871msec

Disk stats (read/write):
sdb: ios=84/264, merge=3/0, ticks=110/240687, in_queue=242880, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/40K /s] [0 /10 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=4359
write: io=1052.0KB, bw=35262 B/s, iops=8, runt= 30549msec
slat (usec): min=4, max=22, avg=11.73, stdev= 1.45
clat (msec): min=153, max=909, avg=462.96, stdev=145.01
lat (msec): min=153, max=909, avg=462.98, stdev=145.01
clat percentiles (msec):
| 1.00th=[ 200], 5.00th=[ 258], 10.00th=[ 281], 20.00th=[ 334],
| 30.00th=[ 379], 40.00th=[ 412], 50.00th=[ 453], 60.00th=[ 494],
| 70.00th=[ 537], 80.00th=[ 578], 90.00th=[ 660], 95.00th=[ 734],
| 99.00th=[ 824], 99.50th=[ 889], 99.90th=[ 914], 99.95th=[ 914],
| 99.99th=[ 914]
bw (KB/s): min= 21, max= 47, per=99.67%, avg=33.89, stdev= 6.47
lat (msec): 250=4.18%, 500=58.17%, 750=33.46%, 1000=4.18%
cpu: usr=0.00%, sys=0.06%, ctx=266, majf=0, minf=0
IO depths: 1=0.4%, 2=0.8%, 4=98.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=263, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1052KB, aggrb=34KB/s, minb=34KB/s, maxb=34KB/s, mint=30549msec, maxt=30549msec

Disk stats (read/write):
sdb: ios=84/262, merge=3/0, ticks=113/120700, in_queue=121243, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/40K /s] [0 /10 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=4389
write: io=1040.0KB, bw=35341 B/s, iops=8, runt= 30133msec
slat (usec): min=5, max=38, avg=13.54, stdev= 2.05
clat (msec): min=68, max=434, avg=231.40, stdev=66.36
lat (msec): min=68, max=434, avg=231.41, stdev=66.36
clat percentiles (msec):
| 1.00th=[ 109], 5.00th=[ 137], 10.00th=[ 151], 20.00th=[ 174],
| 30.00th=[ 194], 40.00th=[ 208], 50.00th=[ 225], 60.00th=[ 239],
| 70.00th=[ 260], 80.00th=[ 281], 90.00th=[ 330], 95.00th=[ 351],
| 99.00th=[ 404], 99.50th=[ 408], 99.90th=[ 437], 99.95th=[ 437],
| 99.99th=[ 437]
bw (KB/s): min= 22, max= 48, per=100.00%, avg=34.09, stdev= 6.12
lat (msec): 100=0.77%, 250=65.00%, 500=34.23%
cpu: usr=0.00%, sys=0.05%, ctx=263, majf=0, minf=0
IO depths: 1=0.4%, 2=99.6%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=260, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1040KB, aggrb=34KB/s, minb=34KB/s, maxb=34KB/s, mint=30133msec, maxt=30133msec

Disk stats (read/write):
sdb: ios=84/258, merge=3/0, ticks=110/59687, in_queue=59927, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/40K /s] [0 /10 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=4419
write: io=1036.0KB, bw=35230 B/s, iops=8, runt= 30112msec
slat (usec): min=13, max=33, avg=16.24, stdev= 2.64
clat (msec): min=24, max=279, avg=116.23, stdev=46.96
lat (msec): min=24, max=279, avg=116.25, stdev=46.96
clat percentiles (msec):
| 1.00th=[ 35], 5.00th=[ 46], 10.00th=[ 56], 20.00th=[ 74],
| 30.00th=[ 92], 40.00th=[ 101], 50.00th=[ 114], 60.00th=[ 127],
| 70.00th=[ 139], 80.00th=[ 155], 90.00th=[ 176], 95.00th=[ 200],
| 99.00th=[ 262], 99.50th=[ 269], 99.90th=[ 281], 99.95th=[ 281],
| 99.99th=[ 281]
bw (KB/s): min= 21, max= 47, per=99.56%, avg=33.85, stdev= 6.35
lat (msec): 50=7.34%, 100=31.66%, 250=59.85%, 500=1.16%
cpu: usr=0.02%, sys=0.04%, ctx=265, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=259, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=1036KB, aggrb=34KB/s, minb=34KB/s, maxb=34KB/s, mint=30112msec, maxt=30112msec

Disk stats (read/write):
sdb: ios=84/257, merge=3/0, ticks=110/29853, in_queue=30003, util=99.98%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [55834K/0K /s] [13.7K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25162
read: io=1592.6MB, bw=54321KB/s, iops=13580, runt= 30012msec
slat (usec): min=2, max=3715, avg=71.91, stdev=384.44
clat (msec): min=11, max=184, avg=75.25, stdev= 6.46
lat (msec): min=11, max=184, avg=75.32, stdev= 6.47
bw (KB/s): min=46128, max=55432, per=99.85%, avg=54239.90, stdev=1183.86
cpu: usr=2.08%, sys=5.65%, ctx=12877, majf=0, minf=536
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=407567/0/0, short=0/0/0

lat (msec): 20=0.03%, 50=0.10%, 100=99.16%, 250=0.71%

Run status group 0 (all jobs):
READ: io=1592.6MB, aggrb=54320KB/s, minb=55624KB/s, maxb=55624KB/s, mint=30012msec, maxt=30012msec

Disk stats (read/write):
sdd: ios=406300/0, merge=7/0, ticks=4261556/0, in_queue=4262475, util=99.75%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=128
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [48656K/0K /s] [11.9K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25422
read: io=1439.5MB, bw=49115KB/s, iops=12278, runt= 30011msec
slat (usec): min=2, max=141, avg= 5.67, stdev= 2.73
clat (usec): min=213, max=110027, avg=10416.41, stdev=5804.65
lat (usec): min=217, max=110030, avg=10422.32, stdev=5804.64
bw (KB/s): min= 0, max=54056, per=32.25%, avg=15840.70, stdev=23083.02
cpu: usr=6.68%, sys=15.09%, ctx=309545, majf=0, minf=151
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=368494/0/0, short=0/0/0
lat (usec): 250=0.01%, 500=0.03%, 750=0.13%, 1000=0.22%
lat (msec): 2=1.38%, 4=8.11%, 10=40.02%, 20=47.88%, 50=1.99%
lat (msec): 100=0.24%, 250=0.01%

Run status group 0 (all jobs):
READ: io=1439.5MB, aggrb=49114KB/s, minb=50293KB/s, maxb=50293KB/s, mint=30011msec, maxt=30011msec

Disk stats (read/write):
sdd: ios=367410/0, merge=5/0, ticks=3824886/0, in_queue=3825505, util=99.76%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=64
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [54562K/0K /s] [13.4K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25203
read: io=1544.5MB, bw=52705KB/s, iops=13176, runt= 30006msec
slat (usec): min=2, max=162, avg= 5.55, stdev= 2.82
clat (usec): min=173, max=28179, avg=4849.37, stdev=2171.93
lat (usec): min=194, max=28185, avg=4855.17, stdev=2171.92
bw (KB/s): min= 0, max=53784, per=49.18%, avg=25921.64, stdev=26461.21
cpu: usr=7.18%, sys=16.42%, ctx=328087, majf=0, minf=87
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued r/w/d: total=395367/0/0, short=0/0/0
lat (usec): 250=0.01%, 500=0.17%, 750=0.64%, 1000=0.99%
lat (msec): 2=5.23%, 4=29.45%, 10=61.81%, 20=1.65%, 50=0.04%

Run status group 0 (all jobs):
READ: io=1544.5MB, aggrb=52705KB/s, minb=53969KB/s, maxb=53969KB/s, mint=30006msec, maxt=30006msec

Disk stats (read/write):
sdd: ios=394198/0, merge=2/0, ticks=1909902/0, in_queue=1910098, util=99.75%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [43970K/0K /s] [10.8K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25226
read: io=1403.3MB, bw=47885KB/s, iops=11971, runt= 30003msec
slat (usec): min=3, max=121, avg= 6.51, stdev= 2.75
clat (usec): min=94, max=9940, avg=2664.11, stdev=1397.42
lat (usec): min=109, max=9945, avg=2670.88, stdev=1397.38
bw (KB/s): min=36160, max=55408, per=100.16%, avg=47963.66, stdev=5087.19
cpu: usr=5.78%, sys=14.95%, ctx=302737, majf=0, minf=55
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued r/w/d: total=359175/0/0, short=0/0/0
lat (usec): 100=0.01%, 250=2.09%, 500=7.79%, 750=4.82%, 1000=2.75%
lat (msec): 2=5.04%, 4=53.76%, 10=23.75%

Run status group 0 (all jobs):
READ: io=1403.3MB, aggrb=47885KB/s, minb=49034KB/s, maxb=49034KB/s, mint=30003msec, maxt=30003msec

Disk stats (read/write):
sdd: ios=358185/0, merge=0/0, ticks=953875/0, in_queue=953911, util=99.74%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [53555K/0K /s] [13.8K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25280
read: io=1413.4MB, bw=48238KB/s, iops=12059, runt= 30002msec
slat (usec): min=4, max=118, avg= 8.38, stdev= 3.34
clat (usec): min=84, max=52237, avg=1316.15, stdev=743.98
lat (usec): min=104, max=52246, avg=1324.79, stdev=744.00
bw (KB/s): min=34160, max=53896, per=99.86%, avg=48168.54, stdev=6020.67
cpu: usr=5.31%, sys=15.42%, ctx=306267, majf=0, minf=39
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=361811/0/0, short=0/0/0
lat (usec): 100=0.02%, 250=5.06%, 500=9.22%, 750=5.69%, 1000=6.44%
lat (msec): 2=57.38%, 4=16.17%, 10=0.01%, 50=0.01%, 100=0.01%

Run status group 0 (all jobs):
READ: io=1413.4MB, aggrb=48238KB/s, minb=49395KB/s, maxb=49395KB/s, mint=30002msec, maxt=30002msec

Disk stats (read/write):
sdd: ios=360786/0, merge=0/0, ticks=475948/0, in_queue=475940, util=99.77%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [50597K/0K /s] [12.4K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25299
read: io=1397.5MB, bw=47698KB/s, iops=11924, runt= 30001msec
slat (usec): min=3, max=143, avg= 8.44, stdev= 3.31
clat (usec): min=64, max=52445, avg=660.16, stdev=445.75
lat (usec): min=73, max=52454, avg=668.87, stdev=445.74
bw (KB/s): min=26688, max=51840, per=100.06%, avg=47725.07, stdev=6097.29
cpu: usr=5.36%, sys=15.34%, ctx=300814, majf=0, minf=31
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=357747/0/0, short=0/0/0
lat (usec): 100=0.06%, 250=7.35%, 500=19.37%, 750=44.03%, 1000=17.18%
lat (msec): 2=11.77%, 4=0.23%, 10=0.01%, 50=0.01%, 100=0.01%

Run status group 0 (all jobs):
READ: io=1397.5MB, aggrb=47698KB/s, minb=48842KB/s, maxb=48842KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=356802/0, merge=0/0, ticks=236589/0, in_queue=236581, util=99.74%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [47525K/0K /s] [11.7K/0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25319
read: io=1313.4MB, bw=44827KB/s, iops=11206, runt= 30001msec
slat (usec): min=3, max=128, avg= 8.73, stdev= 3.41
clat (usec): min=60, max=44760, avg=345.88, stdev=197.72
lat (usec): min=69, max=44769, avg=354.90, stdev=197.62
bw (KB/s): min=37792, max=47280, per=99.96%, avg=44805.83, stdev=2312.44
cpu: usr=5.01%, sys=15.25%, ctx=282863, majf=0, minf=28
IO depths: 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=336211/0/0, short=0/0/0
lat (usec): 100=0.09%, 250=19.26%, 500=71.01%, 750=9.05%, 1000=0.02%
lat (msec): 2=0.55%, 4=0.01%, 10=0.01%, 50=0.01%

Run status group 0 (all jobs):
READ: io=1313.4MB, aggrb=44826KB/s, minb=45902KB/s, maxb=45902KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=335261/0, merge=0/0, ticks=116934/0, in_queue=116921, util=99.75%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [34082K/0K /s] [8321 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25331
read: io=976420KB, bw=32546KB/s, iops=8136, runt= 30001msec
slat (usec): min=4, max=135, avg= 8.94, stdev= 3.14
clat (usec): min=82, max=44081, avg=234.51, stdev=146.58
lat (usec): min=101, max=44089, avg=243.73, stdev=146.52
bw (KB/s): min=27816, max=33888, per=99.96%, avg=32532.88, stdev=1031.17
cpu: usr=3.82%, sys=11.54%, ctx=214442, majf=0, minf=25
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=244105/0/0, short=0/0/0
lat (usec): 100=0.32%, 250=63.16%, 500=36.21%, 750=0.01%, 1000=0.02%
lat (msec): 2=0.28%, 4=0.01%, 10=0.01%, 50=0.01%

Run status group 0 (all jobs):
READ: io=976420KB, aggrb=32546KB/s, minb=33327KB/s, maxb=33327KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=243444/0, merge=0/0, ticks=57717/0, in_queue=57707, util=99.70%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [19505K/0K /s] [4762 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=25357
read: io=580104KB, bw=19336KB/s, iops=4834, runt= 30001msec
slat (usec): min=4, max=103, avg= 8.59, stdev= 2.31
clat (usec): min=52, max=4069, avg=196.02, stdev=45.26
lat (usec): min=58, max=4078, avg=204.88, stdev=45.45
bw (KB/s): min=18624, max=19936, per=100.05%, avg=19346.31, stdev=295.36
cpu: usr=2.32%, sys=6.80%, ctx=145027, majf=0, minf=25
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=145026/0/0, short=0/0/0
lat (usec): 100=0.27%, 250=98.77%, 500=0.85%, 750=0.01%, 1000=0.02%
lat (msec): 2=0.09%, 4=0.01%, 10=0.01%

Run status group 0 (all jobs):
READ: io=580104KB, aggrb=19336KB/s, minb=19800KB/s, maxb=19800KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=144605/0, merge=0/0, ticks=28537/0, in_queue=28528, util=95.12%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/71737K /s] [0 /17.6K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25502
write: io=2082.6MB, bw=71064KB/s, iops=17765, runt= 30008msec
slat (usec): min=2, max=49835, avg=54.11, stdev=424.63
clat (msec): min=7, max=163, avg=57.55, stdev= 9.97
lat (msec): min=7, max=163, avg=57.60, stdev= 9.98
bw (KB/s): min=61960, max=74416, per=100.02%, avg=71074.07, stdev=3099.18
cpu: usr=2.65%, sys=10.56%, ctx=16632, majf=0, minf=23
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=0/533119/0, short=0/0/0

lat (msec): 10=0.01%, 20=0.04%, 50=6.73%, 100=91.88%, 250=1.35%

Run status group 0 (all jobs):
WRITE: io=2082.6MB, aggrb=71063KB/s, minb=72769KB/s, maxb=72769KB/s, mint=30008msec, maxt=30008msec

Disk stats (read/write):
sdd: ios=0/531357, merge=0/14, ticks=0/4256081, in_queue=4256786, util=99.74%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=512
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/71751K /s] [0 /17.6K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25524
write: io=2070.2MB, bw=70673KB/s, iops=17668, runt= 30007msec
slat (usec): min=2, max=48135, avg=54.43, stdev=429.52
clat (msec): min=5, max=138, avg=28.92, stdev= 7.88
lat (msec): min=7, max=138, avg=28.97, stdev= 7.88
bw (KB/s): min=66355, max=73584, per=100.10%, avg=70740.36, stdev=2833.32
cpu: usr=2.69%, sys=10.19%, ctx=16510, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=0/530174/0, short=0/0/0

lat (msec): 10=0.01%, 20=0.17%, 50=97.07%, 100=2.74%, 250=0.01%

Run status group 0 (all jobs):
WRITE: io=2070.2MB, aggrb=70673KB/s, minb=72369KB/s, maxb=72369KB/s, mint=30007msec, maxt=30007msec

Disk stats (read/write):
sdd: ios=0/528541, merge=0/12, ticks=0/4257407, in_queue=4258063, util=99.75%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=256
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/71315K /s] [0 /17.5K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25546
write: io=2072.1MB, bw=70731KB/s, iops=17682, runt= 30010msec
slat (usec): min=2, max=49650, avg=54.45, stdev=426.61
clat (msec): min=2, max=119, avg=14.42, stdev= 6.41
lat (msec): min=2, max=119, avg=14.48, stdev= 6.42
bw (KB/s): min=65992, max=73437, per=100.02%, avg=70748.44, stdev=2926.02
cpu: usr=2.60%, sys=9.86%, ctx=16555, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=0/530661/0, short=0/0/0

lat (msec): 4=0.01%, 10=17.81%, 20=76.24%, 50=4.75%, 100=1.18%
lat (msec): 250=0.01%

Run status group 0 (all jobs):
WRITE: io=2072.1MB, aggrb=70731KB/s, minb=72428KB/s, maxb=72428KB/s, mint=30010msec, maxt=30010msec

Disk stats (read/write):
sdd: ios=0/528998, merge=0/11, ticks=0/4278359, in_queue=4278963, util=99.75%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=128
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/71909K /s] [0 /17.6K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25576
write: io=2068.7MB, bw=70591KB/s, iops=17647, runt= 30008msec
slat (usec): min=2, max=163, avg= 5.88, stdev= 4.40
clat (usec): min=484, max=108710, avg=7244.71, stdev=5168.71
lat (usec): min=487, max=108714, avg=7250.84, stdev=5168.65
bw (KB/s): min=65860, max=73405, per=100.14%, avg=70691.51, stdev=2874.92
cpu: usr=7.44%, sys=17.22%, ctx=387944, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=0/529576/0, short=0/0/0
lat (usec): 500=0.01%, 750=0.01%, 1000=0.31%
lat (msec): 2=5.26%, 4=15.84%, 10=60.46%, 20=16.73%, 50=1.00%
lat (msec): 100=0.39%, 250=0.01%

Run status group 0 (all jobs):
WRITE: io=2068.7MB, aggrb=70591KB/s, minb=72285KB/s, maxb=72285KB/s, mint=30008msec, maxt=30008msec

Disk stats (read/write):
sdd: ios=0/527794, merge=0/8, ticks=0/3821818, in_queue=3822381, util=99.72%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=64
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/71446K /s] [0 /17.5K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25602
write: io=2069.8MB, bw=70635KB/s, iops=17658, runt= 30005msec
slat (usec): min=2, max=160, avg= 5.85, stdev= 4.73
clat (usec): min=487, max=61425, avg=3616.35, stdev=2845.27
lat (usec): min=504, max=61431, avg=3622.46, stdev=2845.16
bw (KB/s): min=66011, max=73784, per=100.03%, avg=70656.08, stdev=2903.02
cpu: usr=7.83%, sys=16.87%, ctx=383027, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued r/w/d: total=0/529849/0, short=0/0/0
lat (usec): 500=0.01%, 750=0.03%, 1000=0.99%
lat (msec): 2=16.88%, 4=46.75%, 10=34.56%, 20=0.51%, 50=0.25%
lat (msec): 100=0.03%

Run status group 0 (all jobs):
WRITE: io=2069.8MB, aggrb=70634KB/s, minb=72329KB/s, maxb=72329KB/s, mint=30005msec, maxt=30005msec

Disk stats (read/write):
sdd: ios=0/528157, merge=0/3, ticks=0/1908453, in_queue=1908547, util=99.74%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/72024K /s] [0 /17.6K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25630
write: io=2064.2MB, bw=70448KB/s, iops=17611, runt= 30003msec
slat (usec): min=3, max=472, avg= 6.86, stdev= 4.85
clat (usec): min=361, max=64001, avg=1808.25, stdev=1900.05
lat (usec): min=381, max=64006, avg=1815.36, stdev=1899.96
bw (KB/s): min=64232, max=73584, per=100.13%, avg=70539.47, stdev=2978.82
cpu: usr=7.51%, sys=16.93%, ctx=376150, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/528410/0, short=0/0/0
lat (usec): 500=0.08%, 750=1.38%, 1000=25.44%
lat (msec): 2=27.79%, 4=44.06%, 10=1.10%, 20=0.01%, 50=0.14%
lat (msec): 100=0.01%

Run status group 0 (all jobs):
WRITE: io=2064.2MB, aggrb=70447KB/s, minb=72138KB/s, maxb=72138KB/s, mint=30003msec, maxt=30003msec

Disk stats (read/write):
sdd: ios=0/526823, merge=0/0, ticks=0/951988, in_queue=951967, util=99.74%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/71696K /s] [0 /17.6K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25666
write: io=2067.4MB, bw=70562KB/s, iops=17640, runt= 30001msec
slat (usec): min=4, max=160, avg= 7.91, stdev= 4.52
clat (usec): min=134, max=48672, avg=897.30, stdev=1390.50
lat (usec): min=143, max=48678, avg=905.46, stdev=1390.18
bw (KB/s): min=65424, max=74296, per=100.03%, avg=70583.68, stdev=2998.17
cpu: usr=6.75%, sys=17.48%, ctx=376455, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=100.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/529233/0, short=0/0/0
lat (usec): 250=0.52%, 500=46.44%, 750=16.68%, 1000=8.85%
lat (msec): 2=14.06%, 4=13.36%, 10=0.01%, 50=0.07%

Run status group 0 (all jobs):
WRITE: io=2067.4MB, aggrb=70562KB/s, minb=72255KB/s, maxb=72255KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=0/527771, merge=0/0, ticks=0/474172, in_queue=474137, util=99.74%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/60592K /s] [0 /14.8K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25718
write: io=1993.1MB, bw=68056KB/s, iops=17014, runt= 30001msec
slat (usec): min=4, max=153, avg= 7.96, stdev= 4.44
clat (usec): min=45, max=55053, avg=460.47, stdev=1040.65
lat (usec): min=50, max=55059, avg=468.67, stdev=1040.38
bw (KB/s): min=47096, max=74344, per=100.44%, avg=68355.95, stdev=6706.70
cpu: usr=6.40%, sys=17.64%, ctx=373251, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/510439/0, short=0/0/0
lat (usec): 50=0.01%, 100=1.25%, 250=65.56%, 500=6.91%, 750=11.48%
lat (usec): 1000=2.55%
lat (msec): 2=9.06%, 4=3.14%, 10=0.02%, 20=0.01%, 50=0.03%
lat (msec): 100=0.01%

Run status group 0 (all jobs):
WRITE: io=1993.1MB, aggrb=68056KB/s, minb=69689KB/s, maxb=69689KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=0/509333, merge=0/0, ticks=0/234860, in_queue=234810, util=99.74%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/69414K /s] [0 /16.1K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25739
write: io=1585.6MB, bw=54118KB/s, iops=13529, runt= 30001msec
slat (usec): min=4, max=148, avg= 8.58, stdev= 3.86
clat (usec): min=22, max=53292, avg=285.05, stdev=732.96
lat (usec): min=36, max=53299, avg=293.89, stdev=732.88
bw (KB/s): min=41992, max=71504, per=99.55%, avg=53871.80, stdev=6853.35
cpu: usr=5.90%, sys=16.46%, ctx=341954, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/405896/0, short=0/0/0
lat (usec): 50=2.69%, 100=43.57%, 250=1.37%, 500=46.21%, 750=1.71%
lat (usec): 1000=0.58%
lat (msec): 2=2.73%, 4=1.05%, 10=0.07%, 20=0.01%, 50=0.02%
lat (msec): 100=0.01%

Run status group 0 (all jobs):
WRITE: io=1585.6MB, aggrb=54117KB/s, minb=55416KB/s, maxb=55416KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=0/404392, merge=0/0, ticks=0/116249, in_queue=116226, util=99.72%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/52981K /s] [0 /12.1K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25799
write: io=1624.6MB, bw=55449KB/s, iops=13862, runt= 30001msec
slat (usec): min=4, max=121, avg= 9.00, stdev= 3.99
clat (usec): min=8, max=47598, avg=133.14, stdev=495.36
lat (usec): min=38, max=47603, avg=142.43, stdev=495.31
bw (KB/s): min=41704, max=74514, per=100.17%, avg=55540.66, stdev=8889.68
cpu: usr=6.80%, sys=17.36%, ctx=355805, majf=0, minf=22
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/415881/0, short=0/0/0
lat (usec): 10=0.01%, 20=0.06%, 50=50.54%, 100=3.83%, 250=42.08%
lat (usec): 500=0.83%, 750=0.57%, 1000=0.13%
lat (msec): 2=1.64%, 4=0.29%, 10=0.02%, 50=0.01%

Run status group 0 (all jobs):
WRITE: io=1624.6MB, aggrb=55448KB/s, minb=56779KB/s, maxb=56779KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=0/414844, merge=0/0, ticks=0/56656, in_queue=56639, util=99.60%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/52297K /s] [0 /12.8K iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=25822
write: io=1532.9MB, bw=52320KB/s, iops=13079, runt= 30001msec
slat (usec): min=5, max=99, avg= 8.85, stdev= 2.53
clat (usec): min=0, max=48971, avg=65.35, stdev=311.65
lat (usec): min=37, max=48981, avg=74.47, stdev=311.67
bw (KB/s): min=41920, max=73824, per=99.98%, avg=52310.10, stdev=6877.76
cpu: usr=6.74%, sys=17.84%, ctx=392397, majf=0, minf=22
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/392410/0, short=0/0/0
lat (usec): 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=47.18%
lat (usec): 100=51.47%, 250=0.59%, 500=0.18%, 750=0.19%, 1000=0.17%
lat (msec): 2=0.18%, 4=0.02%, 10=0.01%, 50=0.01%

Run status group 0 (all jobs):
WRITE: io=1532.9MB, aggrb=52319KB/s, minb=53575KB/s, maxb=53575KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdd: ios=0/391034, merge=0/0, ticks=0/26366, in_queue=26344, util=87.84%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=128
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [0.0% done] [73K/0K /s] [18 /0 iops] [eta 42d:18h:13m:10s]
readtest: (groupid=0, jobs=1): err= 0: pid=26563
read: io=16900KB, bw=558713 B/s, iops=136, runt= 30974msec
slat (usec): min=3, max=124, avg= 9.05, stdev= 4.35
clat (msec): min=17, max=3368, avg=937.95, stdev=589.91
lat (msec): min=18, max=3368, avg=937.96, stdev=589.91
bw (KB/s): min= 0, max= 625, per=31.90%, avg=173.88, stdev=255.59
cpu: usr=0.13%, sys=0.28%, ctx=4225, majf=0, minf=152
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=0.8%, >=64=98.5%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=4225/0/0, short=0/0/0

lat (msec): 20=0.02%, 50=0.26%, 100=1.63%, 250=8.99%, 500=18.11%
lat (msec): 750=16.00%, 1000=13.78%, 2000=36.31%, >=2000=4.90%

Run status group 0 (all jobs):
READ: io=16900KB, aggrb=545KB/s, minb=558KB/s, maxb=558KB/s, mint=30974msec, maxt=30974msec

Disk stats (read/write):
sdb: ios=4213/0, merge=0/0, ticks=3883587/0, in_queue=3900496, util=99.60%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=64
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [540K/0K /s] [132 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26631
read: io=15600KB, bw=518986 B/s, iops=126, runt= 30780msec
slat (usec): min=4, max=45, avg=10.96, stdev= 2.37
clat (msec): min=8, max=2411, avg=504.75, stdev=311.77
lat (msec): min=8, max=2411, avg=504.76, stdev=311.77
bw (KB/s): min= 0, max= 565, per=49.08%, avg=248.34, stdev=256.27
cpu: usr=0.19%, sys=0.28%, ctx=3900, majf=0, minf=87
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=0.8%, >=64=98.4%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued r/w/d: total=3900/0/0, short=0/0/0

lat (msec): 10=0.03%, 50=0.36%, 100=3.44%, 250=18.69%, 500=34.36%
lat (msec): 750=22.77%, 1000=12.59%, 2000=7.74%, >=2000=0.03%

Run status group 0 (all jobs):
READ: io=15600KB, aggrb=506KB/s, minb=518KB/s, maxb=518KB/s, mint=30780msec, maxt=30780msec

Disk stats (read/write):
sdb: ios=3892/0, merge=0/0, ticks=1928873/0, in_queue=1936814, util=99.61%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [511K/0K /s] [124 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26812
read: io=14748KB, bw=499089 B/s, iops=121, runt= 30259msec
slat (usec): min=5, max=36, avg=14.30, stdev= 2.71
clat (msec): min=13, max=2281, avg=262.47, stdev=229.50
lat (msec): min=13, max=2281, avg=262.49, stdev=229.50
bw (KB/s): min= 448, max= 542, per=100.14%, avg=487.68, stdev=20.30
cpu: usr=0.16%, sys=0.31%, ctx=3687, majf=0, minf=55
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=99.2%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued r/w/d: total=3687/0/0, short=0/0/0

lat (msec): 20=0.57%, 50=12.96%, 100=14.94%, 250=29.51%, 500=28.53%
lat (msec): 750=9.79%, 1000=2.52%, 2000=1.11%, >=2000=0.05%

Run status group 0 (all jobs):
READ: io=14748KB, aggrb=487KB/s, minb=499KB/s, maxb=499KB/s, mint=30259msec, maxt=30259msec

Disk stats (read/write):
sdb: ios=3673/0, merge=0/0, ticks=956588/0, in_queue=962866, util=99.62%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [421K/0K /s] [102 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26830
read: io=13144KB, bw=446150 B/s, iops=108, runt= 30168msec
slat (usec): min=5, max=93, avg=17.37, stdev= 3.35
clat (msec): min=8, max=920, avg=146.79, stdev=122.95
lat (msec): min=8, max=921, avg=146.81, stdev=122.95
bw (KB/s): min= 402, max= 496, per=100.24%, avg=436.03, stdev=16.06
cpu: usr=0.14%, sys=0.28%, ctx=3286, majf=0, minf=39
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=99.5%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=3286/0/0, short=0/0/0

lat (msec): 10=0.15%, 20=6.18%, 50=18.72%, 100=19.84%, 250=37.61%
lat (msec): 500=15.86%, 750=1.43%, 1000=0.21%

Run status group 0 (all jobs):
READ: io=13144KB, aggrb=435KB/s, minb=446KB/s, maxb=446KB/s, mint=30168msec, maxt=30168msec

Disk stats (read/write):
sdb: ios=3283/0, merge=0/0, ticks=479629/0, in_queue=481207, util=99.64%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [380K/0K /s] [92 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26848
read: io=11152KB, bw=379415 B/s, iops=92, runt= 30098msec
slat (usec): min=5, max=154, avg=17.66, stdev= 4.06
clat (msec): min=9, max=502, avg=86.29, stdev=67.28
lat (msec): min=9, max=502, avg=86.31, stdev=67.28
bw (KB/s): min= 340, max= 406, per=100.17%, avg=370.64, stdev=15.20
cpu: usr=0.10%, sys=0.25%, ctx=2788, majf=0, minf=31
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=99.7%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=2788/0/0, short=0/0/0

lat (msec): 10=0.04%, 20=6.74%, 50=32.89%, 100=27.19%, 250=30.42%
lat (msec): 500=2.69%, 750=0.04%

Run status group 0 (all jobs):
READ: io=11152KB, aggrb=370KB/s, minb=379KB/s, maxb=379KB/s, mint=30098msec, maxt=30098msec

Disk stats (read/write):
sdb: ios=2770/0, merge=0/0, ticks=238618/0, in_queue=239079, util=99.65%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [299K/0K /s] [73 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26863
read: io=8836.0KB, bw=301030 B/s, iops=73, runt= 30057msec
slat (usec): min=6, max=41, avg=17.78, stdev= 3.12
clat (msec): min=7, max=355, avg=54.37, stdev=38.77
lat (msec): min=7, max=355, avg=54.39, stdev=38.77
bw (KB/s): min= 193, max= 331, per=100.53%, avg=294.55, stdev=20.61
cpu: usr=0.09%, sys=0.20%, ctx=2209, majf=0, minf=27
IO depths: 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=2209/0/0, short=0/0/0

lat (msec): 10=0.14%, 20=5.57%, 50=55.82%, 100=27.89%, 250=10.10%
lat (msec): 500=0.50%

Run status group 0 (all jobs):
READ: io=8836KB, aggrb=293KB/s, minb=301KB/s, maxb=301KB/s, mint=30057msec, maxt=30057msec

Disk stats (read/write):
sdb: ios=2196/0, merge=0/0, ticks=119319/0, in_queue=119501, util=99.63%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [290K/0K /s] [71 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26875
read: io=8148.0KB, bw=278016 B/s, iops=67, runt= 30011msec
slat (usec): min=7, max=41, avg=18.08, stdev= 2.98
clat (msec): min=4, max=249, avg=29.43, stdev=24.66
lat (msec): min=4, max=249, avg=29.45, stdev=24.66
bw (KB/s): min= 246, max= 309, per=100.03%, avg=271.07, stdev=12.89
cpu: usr=0.09%, sys=0.19%, ctx=2037, majf=0, minf=26
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=2037/0/0, short=0/0/0

lat (msec): 10=9.33%, 20=38.49%, 50=37.60%, 100=12.37%, 250=2.21%

Run status group 0 (all jobs):
READ: io=8148KB, aggrb=271KB/s, minb=278KB/s, maxb=278KB/s, mint=30011msec, maxt=30011msec

Disk stats (read/write):
sdb: ios=2027/0, merge=0/0, ticks=59697/0, in_queue=59716, util=99.60%

root@host# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [266K/0K /s] [65 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=26893
read: io=7564.0KB, bw=258021 B/s, iops=62, runt= 30019msec
slat (usec): min=9, max=39, avg=17.66, stdev= 3.15
clat (msec): min=2, max=250, avg=15.85, stdev= 6.91
lat (msec): min=2, max=250, avg=15.87, stdev= 6.91
bw (KB/s): min= 167, max= 291, per=100.32%, avg=251.80, stdev=18.56
cpu: usr=0.09%, sys=0.16%, ctx=1891, majf=0, minf=24
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=1891/0/0, short=0/0/0

lat (msec): 4=0.16%, 10=9.62%, 20=73.61%, 50=16.55%, 500=0.05%

Run status group 0 (all jobs):
READ: io=7564KB, aggrb=251KB/s, minb=258KB/s, maxb=258KB/s, mint=30019msec, maxt=30019msec

Disk stats (read/write):
sdb: ios=1883/0, merge=0/0, ticks=29818/0, in_queue=29829, util=99.45%

root@home# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/527K /s] [0 /128 iops] [eta 45d:14h:53m:02s]
writetest: (groupid=0, jobs=1): err= 0: pid=26924
write: io=19456KB, bw=643734 B/s, iops=157, runt= 30949msec
slat (usec): min=2, max=282743, avg=6210.03, stdev=35626.77
clat (msec): min=727, max=11001, avg=6071.14, stdev=1458.55
lat (msec): min=962, max=11001, avg=6077.35, stdev=1458.99
bw (KB/s): min= 0, max= 728, per=3.71%, avg=23.31, stdev=118.34
cpu: usr=0.06%, sys=0.07%, ctx=276, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.3%, 32=0.7%, >=64=98.7%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued r/w/d: total=0/4864/0, short=0/0/0

lat (msec): 750=0.02%, 1000=0.66%, 2000=3.29%, >=2000=96.03%

Run status group 0 (all jobs):
WRITE: io=19456KB, aggrb=628KB/s, minb=643KB/s, maxb=643KB/s, mint=30949msec, maxt=30949msec

Disk stats (read/write):
sdb: ios=83/4855, merge=0/0, ticks=131/4370631, in_queue=4378949, util=100.00%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/581K /s] [0 /141 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=26956
write: io=17380KB, bw=589269 B/s, iops=143, runt= 30202msec
slat (usec): min=4, max=64, avg=15.65, stdev= 2.85
clat (usec): min=497, max=398588, avg=222319.82, stdev=37626.92
lat (usec): min=519, max=398600, avg=222335.96, stdev=37627.65
bw (KB/s): min= 486, max= 1093, per=99.97%, avg=574.85, stdev=77.20
cpu: usr=0.21%, sys=0.36%, ctx=4345, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=99.3%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/4345/0, short=0/0/0
lat (usec): 500=0.02%
lat (msec): 2=0.14%, 4=0.30%, 10=1.17%, 50=0.09%, 100=0.16%
lat (msec): 250=83.87%, 500=14.25%

Run status group 0 (all jobs):
WRITE: io=17380KB, aggrb=575KB/s, minb=589KB/s, maxb=589KB/s, mint=30202msec, maxt=30202msec

Disk stats (read/write):
sdb: ios=83/4336, merge=0/0, ticks=64/961069, in_queue=962665, util=99.84%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/573K /s] [0 /140 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=26980
write: io=17364KB, bw=590036 B/s, iops=144, runt= 30135msec
slat (usec): min=4, max=53, avg=19.06, stdev= 3.23
clat (usec): min=355, max=222467, avg=110994.39, stdev=20437.11
lat (usec): min=381, max=222490, avg=111013.98, stdev=20437.75
bw (KB/s): min= 491, max= 1109, per=100.16%, avg=576.90, stdev=78.30
cpu: usr=0.14%, sys=0.43%, ctx=4342, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=99.7%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/4341/0, short=0/0/0
lat (usec): 500=0.02%, 1000=0.02%
lat (msec): 2=0.16%, 4=1.41%, 10=0.02%, 50=0.12%, 100=19.79%
lat (msec): 250=78.46%

Run status group 0 (all jobs):
WRITE: io=17364KB, aggrb=576KB/s, minb=590KB/s, maxb=590KB/s, mint=30135msec, maxt=30135msec

Disk stats (read/write):
sdb: ios=83/4341, merge=0/0, ticks=67/480952, in_queue=481017, util=99.84%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/626K /s] [0 /152 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=27008
write: io=17160KB, bw=584675 B/s, iops=142, runt= 30054msec
slat (usec): min=5, max=44, avg=19.01, stdev= 3.15
clat (usec): min=278, max=95168, avg=55996.99, stdev=11865.20
lat (usec): min=307, max=95187, avg=56016.54, stdev=11865.82
bw (KB/s): min= 458, max= 1099, per=100.08%, avg=570.47, stdev=79.55
cpu: usr=0.17%, sys=0.40%, ctx=4291, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=99.8%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/4290/0, short=0/0/0
lat (usec): 500=0.02%, 750=0.02%, 1000=0.02%
lat (msec): 2=1.56%, 4=0.02%, 20=0.02%, 50=25.78%, 100=72.54%

Run status group 0 (all jobs):
WRITE: io=17160KB, aggrb=570KB/s, minb=584KB/s, maxb=584KB/s, mint=30054msec, maxt=30054msec

Disk stats (read/write):
sdb: ios=83/4264, merge=0/0, ticks=59/238810, in_queue=239070, util=99.83%

root@home# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/573K /s] [0 /140 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=27030
write: io=17100KB, bw=583096 B/s, iops=142, runt= 30030msec
slat (usec): min=5, max=37, avg=18.82, stdev= 3.02
clat (usec): min=274, max=53388, avg=28063.83, stdev=6724.53
lat (usec): min=296, max=53410, avg=28083.17, stdev=6725.00
bw (KB/s): min= 492, max= 1109, per=100.14%, avg=569.81, stdev=78.93
cpu: usr=0.17%, sys=0.39%, ctx=4275, majf=0, minf=22
IO depths: 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/4275/0, short=0/0/0
lat (usec): 500=0.02%, 750=1.54%, 1000=0.07%
lat (msec): 4=0.02%, 20=5.38%, 50=92.82%, 100=0.14%

Run status group 0 (all jobs):
WRITE: io=17100KB, aggrb=569KB/s, minb=583KB/s, maxb=583KB/s, mint=30030msec, maxt=30030msec

Disk stats (read/write):
sdb: ios=83/4256, merge=0/0, ticks=60/119372, in_queue=119517, util=99.82%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/626K /s] [0 /152 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=27064
write: io=17172KB, bw=585825 B/s, iops=143, runt= 30016msec
slat (usec): min=6, max=40, avg=18.67, stdev= 3.01
clat (usec): min=158, max=27959, avg=13954.43, stdev=3872.39
lat (usec): min=186, max=27980, avg=13973.63, stdev=3872.96
bw (KB/s): min= 476, max= 1087, per=99.98%, avg=571.86, stdev=78.33
cpu: usr=0.21%, sys=0.36%, ctx=4293, majf=0, minf=22
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/4293/0, short=0/0/0
lat (usec): 250=0.02%, 500=1.54%, 750=0.09%
lat (msec): 10=10.16%, 20=84.30%, 50=3.89%

Run status group 0 (all jobs):
WRITE: io=17172KB, aggrb=572KB/s, minb=585KB/s, maxb=585KB/s, mint=30016msec, maxt=30016msec

Disk stats (read/write):
sdb: ios=83/4273, merge=0/0, ticks=64/59670, in_queue=59743, util=99.80%

root@host# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
fio 1.57
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/569K /s] [0 /139 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=27088
write: io=17020KB, bw=580929 B/s, iops=141, runt= 30001msec
slat (usec): min=6, max=74, avg=18.97, stdev= 3.19
clat (usec): min=137, max=18822, avg=7025.40, stdev=2379.35
lat (usec): min=151, max=18841, avg=7044.92, stdev=2379.66
bw (KB/s): min= 459, max= 1117, per=100.04%, avg=567.25, stdev=81.49
cpu: usr=0.18%, sys=0.38%, ctx=4255, majf=0, minf=22
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued r/w/d: total=0/4255/0, short=0/0/0
lat (usec): 250=1.60%, 500=0.07%
lat (msec): 2=0.05%, 4=5.69%, 10=87.99%, 20=4.61%

Run status group 0 (all jobs):
WRITE: io=17020KB, aggrb=567KB/s, minb=580KB/s, maxb=580KB/s, mint=30001msec, maxt=30001msec

Disk stats (read/write):
sdb: ios=83/4235, merge=0/0, ticks=62/29802, in_queue=29863, util=99.56%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [0.0% done] [516K/0K /s] [129 /0 iops] [eta 06d:06h:39m:59s]
readtest: (groupid=0, jobs=1): err= 0: pid=3813
read: io=18048KB, bw=598599 B/s, iops=146, runt= 30874msec
slat (usec): min=2, max=287017, avg=6648.57, stdev=37692.92
clat (msec): min=863, max=9575, avg=6409.46, stdev=1605.46
lat (msec): min=1077, max=9575, avg=6416.11, stdev=1605.43
clat percentiles (msec):
| 1.00th=[ 1303], 5.00th=[ 2573], 10.00th=[ 4080], 20.00th=[ 6128],
| 30.00th=[ 6194], 40.00th=[ 6325], 50.00th=[ 6587], 60.00th=[ 6783],
| 70.00th=[ 7177], 80.00th=[ 7635], 90.00th=[ 8160], 95.00th=[ 8356],
| 99.00th=[ 8717], 99.50th=[ 8848], 99.90th=[ 9110], 99.95th=[ 9372],
| 99.99th=[ 9634]
bw (KB/s): min= 0, max= 644, per=95.81%, avg=559.53, stdev=117.13
lat (msec): 1000=0.02%, 2000=3.55%, >=2000=96.43%
cpu: usr=0.03%, sys=0.08%, ctx=265, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=0.7%, >=64=98.6%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=4512/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=18048KB, aggrb=584KB/s, minb=584KB/s, maxb=584KB/s, mint=30874msec, maxt=30874msec

Disk stats (read/write):
sda: ios=4482/0, merge=0/0, ticks=4311286/0, in_queue=4353880, util=99.73%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [472K/0K /s] [118 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3826
read: io=15136KB, bw=512118 B/s, iops=125, runt= 30265msec
slat (usec): min=4, max=129, avg=23.86, stdev= 4.10
clat (msec): min=8, max=1241, avg=255.83, stdev=212.30
lat (msec): min=8, max=1241, avg=255.86, stdev=212.30
clat percentiles (msec):
| 1.00th=[ 21], 5.00th=[ 34], 10.00th=[ 45], 20.00th=[ 71],
| 30.00th=[ 99], 40.00th=[ 133], 50.00th=[ 180], 60.00th=[ 247],
| 70.00th=[ 338], 80.00th=[ 449], 90.00th=[ 586], 95.00th=[ 685],
| 99.00th=[ 840], 99.50th=[ 889], 99.90th=[ 996], 99.95th=[ 1045],
| 99.99th=[ 1237]
bw (KB/s): min= 438, max= 538, per=100.00%, avg=500.36, stdev=23.40
lat (msec): 10=0.03%, 20=0.79%, 50=11.44%, 100=18.02%, 250=30.02%
lat (msec): 500=23.65%, 750=13.48%, 1000=2.51%, 2000=0.05%
cpu: usr=0.31%, sys=0.48%, ctx=3784, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.4%, 32=99.2%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=3784/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=15136KB, aggrb=500KB/s, minb=500KB/s, maxb=500KB/s, mint=30265msec, maxt=30265msec

Disk stats (read/write):
sda: ios=3771/0, merge=0/0, ticks=958434/0, in_queue=963377, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [460K/0K /s] [115 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3832
read: io=13964KB, bw=474046 B/s, iops=115, runt= 30164msec
slat (usec): min=4, max=48, avg= 9.72, stdev= 2.26
clat (msec): min=11, max=677, avg=138.18, stdev=104.64
lat (msec): min=11, max=677, avg=138.19, stdev=104.64
clat percentiles (msec):
| 1.00th=[ 21], 5.00th=[ 29], 10.00th=[ 37], 20.00th=[ 55],
| 30.00th=[ 72], 40.00th=[ 90], 50.00th=[ 108], 60.00th=[ 129],
| 70.00th=[ 159], 80.00th=[ 210], 90.00th=[ 289], 95.00th=[ 359],
| 99.00th=[ 474], 99.50th=[ 545], 99.90th=[ 652], 99.95th=[ 660],
| 99.99th=[ 676]
bw (KB/s): min= 422, max= 507, per=100.00%, avg=463.22, stdev=21.17
lat (msec): 20=0.92%, 50=16.47%, 100=28.87%, 250=39.01%, 500=14.01%
lat (msec): 750=0.72%
cpu: usr=0.16%, sys=0.20%, ctx=3491, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=99.6%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=3491/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=13964KB, aggrb=462KB/s, minb=462KB/s, maxb=462KB/s, mint=30164msec, maxt=30164msec

Disk stats (read/write):
sda: ios=3490/0, merge=0/0, ticks=481087/0, in_queue=481253, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [460K/0K /s] [115 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3832
read: io=13964KB, bw=474046 B/s, iops=115, runt= 30164msec
slat (usec): min=4, max=48, avg= 9.72, stdev= 2.26
clat (msec): min=11, max=677, avg=138.18, stdev=104.64
lat (msec): min=11, max=677, avg=138.19, stdev=104.64
clat percentiles (msec):
| 1.00th=[ 21], 5.00th=[ 29], 10.00th=[ 37], 20.00th=[ 55],
| 30.00th=[ 72], 40.00th=[ 90], 50.00th=[ 108], 60.00th=[ 129],
| 70.00th=[ 159], 80.00th=[ 210], 90.00th=[ 289], 95.00th=[ 359],
| 99.00th=[ 474], 99.50th=[ 545], 99.90th=[ 652], 99.95th=[ 660],
| 99.99th=[ 676]
bw (KB/s): min= 422, max= 507, per=100.00%, avg=463.22, stdev=21.17
lat (msec): 20=0.92%, 50=16.47%, 100=28.87%, 250=39.01%, 500=14.01%
lat (msec): 750=0.72%
cpu: usr=0.16%, sys=0.20%, ctx=3491, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=99.6%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=3491/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=13964KB, aggrb=462KB/s, minb=462KB/s, maxb=462KB/s, mint=30164msec, maxt=30164msec

Disk stats (read/write):
sda: ios=3490/0, merge=0/0, ticks=481087/0, in_queue=481253, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [448K/0K /s] [112 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3836
read: io=13024KB, bw=443783 B/s, iops=108, runt= 30052msec
slat (usec): min=4, max=47, avg= 9.73, stdev= 1.99
clat (msec): min=7, max=211, avg=73.81, stdev=31.56
lat (msec): min=7, max=211, avg=73.82, stdev=31.56
clat percentiles (msec):
| 1.00th=[ 21], 5.00th=[ 27], 10.00th=[ 33], 20.00th=[ 43],
| 30.00th=[ 52], 40.00th=[ 63], 50.00th=[ 74], 60.00th=[ 83],
| 70.00th=[ 93], 80.00th=[ 104], 90.00th=[ 117], 95.00th=[ 126],
| 99.00th=[ 143], 99.50th=[ 149], 99.90th=[ 178], 99.95th=[ 180],
| 99.99th=[ 212]
bw (KB/s): min= 391, max= 463, per=100.00%, avg=433.20, stdev=16.73
lat (msec): 10=0.03%, 20=0.71%, 50=27.12%, 100=49.57%, 250=22.57%
cpu: usr=0.12%, sys=0.22%, ctx=3257, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=99.8%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=3256/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=13024KB, aggrb=433KB/s, minb=433KB/s, maxb=433KB/s, mint=30052msec, maxt=30052msec

Disk stats (read/write):
sda: ios=3240/0, merge=0/0, ticks=238833/0, in_queue=239203, util=99.70%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [356K/0K /s] [89 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3840
read: io=10616KB, bw=361840 B/s, iops=88, runt= 30043msec
slat (usec): min=5, max=46, avg= 9.62, stdev= 1.88
clat (msec): min=12, max=102, avg=45.25, stdev=12.46
lat (msec): min=13, max=102, avg=45.26, stdev=12.46
clat percentiles (msec):
| 1.00th=[ 23], 5.00th=[ 28], 10.00th=[ 30], 20.00th=[ 35],
| 30.00th=[ 38], 40.00th=[ 42], 50.00th=[ 45], 60.00th=[ 48],
| 70.00th=[ 51], 80.00th=[ 56], 90.00th=[ 63], 95.00th=[ 68],
| 99.00th=[ 80], 99.50th=[ 84], 99.90th=[ 97], 99.95th=[ 99],
| 99.99th=[ 103]
bw (KB/s): min= 322, max= 395, per=100.00%, avg=353.08, stdev=17.19
lat (msec): 20=0.26%, 50=66.84%, 100=32.86%, 250=0.04%
cpu: usr=0.18%, sys=0.11%, ctx=2654, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=2654/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=10616KB, aggrb=353KB/s, minb=353KB/s, maxb=353KB/s, mint=30043msec, maxt=30043msec

Disk stats (read/write):
sda: ios=2642/0, merge=0/0, ticks=119520/0, in_queue=119583, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [320K/0K /s] [80 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3844
read: io=10296KB, bw=351144 B/s, iops=85, runt= 30025msec
slat (usec): min=8, max=46, avg=20.46, stdev= 5.06
clat (msec): min=7, max=42, avg=23.29, stdev= 5.06
lat (msec): min=7, max=42, avg=23.32, stdev= 5.06
clat percentiles (usec):
| 1.00th=[11968], 5.00th=[15168], 10.00th=[16768], 20.00th=[19072],
| 30.00th=[20608], 40.00th=[21888], 50.00th=[23168], 60.00th=[24448],
| 70.00th=[25984], 80.00th=[27520], 90.00th=[30080], 95.00th=[31872],
| 99.00th=[35584], 99.50th=[37120], 99.90th=[39168], 99.95th=[39168],
| 99.99th=[42240]
bw (KB/s): min= 314, max= 415, per=100.00%, avg=342.71, stdev=18.16
lat (msec): 10=0.16%, 20=25.41%, 50=74.44%
cpu: usr=0.14%, sys=0.35%, ctx=2574, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=2574/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=10296KB, aggrb=342KB/s, minb=342KB/s, maxb=342KB/s, mint=30025msec, maxt=30025msec

Disk stats (read/write):
sda: ios=2563/0, merge=0/0, ticks=59727/0, in_queue=59753, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [328K/0K /s] [82 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3852
read: io=10236KB, bw=349284 B/s, iops=85, runt= 30009msec
slat (usec): min=8, max=50, avg=31.99, stdev= 3.37
clat (msec): min=2, max=22, avg=11.69, stdev= 3.49
lat (msec): min=2, max=22, avg=11.72, stdev= 3.49
clat percentiles (usec):
| 1.00th=[ 3984], 5.00th=[ 5920], 10.00th=[ 7200], 20.00th=[ 8512],
| 30.00th=[ 9792], 40.00th=[10816], 50.00th=[11584], 60.00th=[12608],
| 70.00th=[13504], 80.00th=[14784], 90.00th=[16320], 95.00th=[17536],
| 99.00th=[19584], 99.50th=[20096], 99.90th=[21376], 99.95th=[21376],
| 99.99th=[22400]
bw (KB/s): min= 310, max= 390, per=100.00%, avg=341.07, stdev=16.19
lat (msec): 4=1.02%, 10=31.11%, 20=67.37%, 50=0.51%
cpu: usr=0.06%, sys=0.41%, ctx=2559, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=2559/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=10236KB, aggrb=341KB/s, minb=341KB/s, maxb=341KB/s, mint=30009msec, maxt=30009msec

Disk stats (read/write):
sda: ios=2552/0, merge=0/0, ticks=29780/0, in_queue=29786, util=99.29%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/644K /s] [0 /161 iops] [eta 04d:10h:34m:55s]
writetest: (groupid=0, jobs=1): err= 0: pid=3862
write: io=23820KB, bw=792709 B/s, iops=193, runt= 30770msec
slat (usec): min=3, max=226929, avg=5052.81, stdev=28899.87
clat (msec): min=665, max=7288, avg=5036.55, stdev=1103.42
lat (msec): min=877, max=7288, avg=5041.60, stdev=1103.79
clat percentiles (msec):
| 1.00th=[ 1057], 5.00th=[ 2376], 10.00th=[ 3982], 20.00th=[ 4686],
| 30.00th=[ 4817], 40.00th=[ 4948], 50.00th=[ 5145], 60.00th=[ 5276],
| 70.00th=[ 5538], 80.00th=[ 5866], 90.00th=[ 6194], 95.00th=[ 6390],
| 99.00th=[ 6718], 99.50th=[ 6783], 99.90th=[ 6980], 99.95th=[ 7111],
| 99.99th=[ 7308]
bw (KB/s): min= 1, max= 850, per=93.87%, avg=726.55, stdev=140.26
lat (msec): 750=0.02%, 1000=0.54%, 2000=3.22%, >=2000=96.22%
cpu: usr=0.03%, sys=0.12%, ctx=311, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.3%, 32=0.5%, >=64=98.9%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=0/w=0/d=5955, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=23820KB, aggrb=774KB/s, minb=774KB/s, maxb=774KB/s, mint=30770msec, maxt=30770msec

Disk stats (read/write):
sda: ios=84/5933, merge=0/0, ticks=230/4329180, in_queue=4359370, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/736K /s] [0 /184 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3885
write: io=23056KB, bw=782310 B/s, iops=190, runt= 30179msec
slat (usec): min=3, max=132, avg=19.51, stdev= 8.95
clat (usec): min=356, max=866732, avg=167484.47, stdev=145167.58
lat (usec): min=361, max=866744, avg=167504.66, stdev=145168.06
clat percentiles (msec):
| 1.00th=[ 4], 5.00th=[ 18], 10.00th=[ 28], 20.00th=[ 48],
| 30.00th=[ 71], 40.00th=[ 93], 50.00th=[ 117], 60.00th=[ 151],
| 70.00th=[ 210], 80.00th=[ 277], 90.00th=[ 396], 95.00th=[ 474],
| 99.00th=[ 603], 99.50th=[ 635], 99.90th=[ 725], 99.95th=[ 791],
| 99.99th=[ 865]
bw (KB/s): min= 646, max= 1865, per=100.00%, avg=764.76, stdev=150.29
lat (usec): 500=0.05%, 750=0.03%, 1000=0.07%
lat (msec): 2=0.33%, 4=0.57%, 10=1.56%, 20=3.54%, 50=14.50%
lat (msec): 100=23.30%, 250=32.43%, 500=19.69%, 750=3.85%, 1000=0.07%
cpu: usr=0.24%, sys=0.69%, ctx=5767, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.3%, 32=99.5%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=5764, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=23056KB, aggrb=763KB/s, minb=763KB/s, maxb=763KB/s, mint=30179msec, maxt=30179msec

Disk stats (read/write):
sda: ios=84/5760, merge=0/0, ticks=223/960820, in_queue=962817, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/712K /s] [0 /178 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3907
write: io=22592KB, bw=769140 B/s, iops=187, runt= 30078msec
slat (usec): min=3, max=43, avg=19.45, stdev= 8.87
clat (usec): min=355, max=515043, avg=85163.46, stdev=55854.61
lat (usec): min=360, max=515074, avg=85183.85, stdev=55855.21
clat percentiles (usec):
| 1.00th=[ 1720], 5.00th=[15936], 10.00th=[23424], 20.00th=[37120],
| 30.00th=[50432], 40.00th=[64256], 50.00th=[78336], 60.00th=[91648],
| 70.00th=[105984], 80.00th=[122368], 90.00th=[150528], 95.00th=[183296],
| 99.00th=[284672], 99.50th=[313344], 99.90th=[391168], 99.95th=[440320],
| 99.99th=[514048]
bw (KB/s): min= 576, max= 1860, per=100.00%, avg=751.53, stdev=151.70
lat (usec): 500=0.05%, 750=0.19%, 1000=0.19%
lat (msec): 2=0.80%, 4=1.10%, 10=0.30%, 20=4.98%, 50=22.17%
lat (msec): 100=36.03%, 250=32.74%, 500=1.43%, 750=0.02%
cpu: usr=0.26%, sys=0.65%, ctx=5649, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=99.7%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=5648, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=22592KB, aggrb=751KB/s, minb=751KB/s, maxb=751KB/s, mint=30078msec, maxt=30078msec

Disk stats (read/write):
sda: ios=84/5613, merge=0/0, ticks=210/477223, in_queue=478567, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/708K /s] [0 /177 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3929
write: io=22908KB, bw=780963 B/s, iops=190, runt= 30037msec
slat (usec): min=3, max=133, avg=16.44, stdev= 7.30
clat (usec): min=356, max=166901, avg=41927.87, stdev=19827.30
lat (usec): min=360, max=166930, avg=41946.17, stdev=19827.88
clat percentiles (usec):
| 1.00th=[ 932], 5.00th=[13888], 10.00th=[17280], 20.00th=[23168],
| 30.00th=[29568], 40.00th=[35072], 50.00th=[40704], 60.00th=[46848],
| 70.00th=[52992], 80.00th=[59648], 90.00th=[68096], 95.00th=[75264],
| 99.00th=[88576], 99.50th=[95744], 99.90th=[113152], 99.95th=[116224],
| 99.99th=[166912]
bw (KB/s): min= 632, max= 1936, per=100.00%, avg=763.63, stdev=161.74
lat (usec): 500=0.30%, 750=0.40%, 1000=0.44%
lat (msec): 2=1.10%, 4=0.23%, 10=0.16%, 20=11.96%, 50=50.29%
lat (msec): 100=34.78%, 250=0.35%
cpu: usr=0.31%, sys=0.61%, ctx=5728, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=99.9%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=5727, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=22908KB, aggrb=762KB/s, minb=762KB/s, maxb=762KB/s, mint=30037msec, maxt=30037msec

Disk stats (read/write):
sda: ios=84/5703, merge=0/0, ticks=96/238850, in_queue=239140, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/768K /s] [0 /192 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3951
write: io=22676KB, bw=773543 B/s, iops=188, runt= 30018msec
slat (usec): min=3, max=44, avg=15.74, stdev= 6.49
clat (usec): min=306, max=66557, avg=21153.83, stdev=7431.97
lat (usec): min=335, max=66586, avg=21171.31, stdev=7432.45
clat percentiles (usec):
| 1.00th=[ 540], 5.00th=[12352], 10.00th=[13632], 20.00th=[15424],
| 30.00th=[17280], 40.00th=[18816], 50.00th=[20352], 60.00th=[22144],
| 70.00th=[23936], 80.00th=[26752], 90.00th=[30592], 95.00th=[34048],
| 99.00th=[41728], 99.50th=[45824], 99.90th=[52992], 99.95th=[55552],
| 99.99th=[66048]
bw (KB/s): min= 630, max= 1904, per=100.00%, avg=756.66, stdev=156.44
lat (usec): 500=0.88%, 750=1.18%, 1000=0.37%
lat (msec): 2=0.11%, 4=0.02%, 10=0.19%, 20=44.47%, 50=52.55%
lat (msec): 100=0.23%
cpu: usr=0.33%, sys=0.56%, ctx=5671, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=5669, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=22676KB, aggrb=755KB/s, minb=755KB/s, maxb=755KB/s, mint=30018msec, maxt=30018msec

Disk stats (read/write):
sda: ios=84/5652, merge=0/0, ticks=100/119303, in_queue=119453, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/724K /s] [0 /181 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3974
write: io=22544KB, bw=769245 B/s, iops=187, runt= 30010msec
slat (usec): min=4, max=46, avg=18.56, stdev= 7.38
clat (usec): min=157, max=28381, avg=10622.52, stdev=3481.52
lat (usec): min=190, max=28406, avg=10642.92, stdev=3482.38
clat percentiles (usec):
| 1.00th=[ 290], 5.00th=[ 7200], 10.00th=[ 7904], 20.00th=[ 8640],
| 30.00th=[ 9024], 40.00th=[ 9536], 50.00th=[ 9920], 60.00th=[10432],
| 70.00th=[11072], 80.00th=[12096], 90.00th=[16320], 95.00th=[18048],
| 99.00th=[20096], 99.50th=[20608], 99.90th=[25472], 99.95th=[26496],
| 99.99th=[28288]
bw (KB/s): min= 664, max= 1920, per=100.00%, avg=751.66, stdev=158.62
lat (usec): 250=0.71%, 500=1.81%, 750=0.05%
lat (msec): 2=0.04%, 10=48.49%, 20=47.91%, 50=0.99%
cpu: usr=0.25%, sys=0.66%, ctx=5639, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=5636, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=22544KB, aggrb=751KB/s, minb=751KB/s, maxb=751KB/s, mint=30010msec, maxt=30010msec

Disk stats (read/write):
sda: ios=84/5618, merge=0/0, ticks=97/59580, in_queue=59680, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/768K /s] [0 /192 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3997
write: io=22532KB, bw=768989 B/s, iops=187, runt= 30004msec
slat (usec): min=6, max=182, avg=26.05, stdev=11.19
clat (usec): min=113, max=17523, avg=5294.30, stdev=2342.17
lat (usec): min=120, max=17539, avg=5321.15, stdev=2342.65
clat percentiles (usec):
| 1.00th=[ 141], 5.00th=[ 2864], 10.00th=[ 3472], 20.00th=[ 3856],
| 30.00th=[ 4320], 40.00th=[ 4576], 50.00th=[ 4896], 60.00th=[ 5280],
| 70.00th=[ 5600], 80.00th=[ 6112], 90.00th=[ 7264], 95.00th=[11712],
| 99.00th=[13760], 99.50th=[14144], 99.90th=[16512], 99.95th=[16768],
| 99.99th=[17536]
bw (KB/s): min= 624, max= 1892, per=100.00%, avg=751.07, stdev=156.08
lat (usec): 250=2.52%, 500=0.04%, 750=0.04%, 1000=0.04%
lat (msec): 2=0.25%, 4=20.34%, 10=71.10%, 20=5.68%
cpu: usr=0.12%, sys=0.66%, ctx=5636, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=5633, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=22532KB, aggrb=750KB/s, minb=750KB/s, maxb=750KB/s, mint=30004msec, maxt=30004msec

Disk stats (read/write):
sda: ios=84/5613, merge=0/0, ticks=100/29717, in_queue=29817, util=99.39%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [0.0% done] [260K/0K /s] [65 /0 iops] [eta 11d:12h:33m:43s]
readtest: (groupid=0, jobs=1): err= 0: pid=3795
read: io=13824KB, bw=452666 B/s, iops=110, runt= 31272msec
slat (usec): min=3, max=411507, avg=8701.29, stdev=49676.88
clat (msec): min=1190, max=12289, avg=8229.64, stdev=2180.39
lat (msec): min=1445, max=12289, avg=8238.34, stdev=2180.71
clat percentiles (msec):
| 1.00th=[ 1860], 5.00th=[ 3064], 10.00th=[ 4490], 20.00th=[ 7767],
| 30.00th=[ 8029], 40.00th=[ 8225], 50.00th=[ 8586], 60.00th=[ 8848],
| 70.00th=[ 9372], 80.00th=[ 9896], 90.00th=[10552], 95.00th=[10814],
| 99.00th=[11207], 99.50th=[11600], 99.90th=[11994], 99.95th=[11994],
| 99.99th=[12256]
bw (KB/s): min= 0, max= 506, per=96.51%, avg=426.56, stdev=97.53
lat (msec): 2000=1.88%, >=2000=98.12%
cpu: usr=0.02%, sys=0.07%, ctx=232, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.5%, 32=0.9%, >=64=98.2%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=3456/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=13824KB, aggrb=442KB/s, minb=442KB/s, maxb=442KB/s, mint=31272msec, maxt=31272msec

Disk stats (read/write):
sda: ios=3444/0, merge=0/0, ticks=4365220/0, in_queue=4391680, util=99.72%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [388K/0K /s] [97 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3799
read: io=11004KB, bw=370844 B/s, iops=90, runt= 30385msec
slat (usec): min=4, max=140, avg=18.36, stdev= 3.36
clat (msec): min=15, max=1357, avg=353.27, stdev=264.64
lat (msec): min=15, max=1357, avg=353.29, stdev=264.64
clat percentiles (msec):
| 1.00th=[ 29], 5.00th=[ 47], 10.00th=[ 66], 20.00th=[ 106],
| 30.00th=[ 149], 40.00th=[ 208], 50.00th=[ 277], 60.00th=[ 375],
| 70.00th=[ 494], 80.00th=[ 611], 90.00th=[ 750], 95.00th=[ 840],
| 99.00th=[ 1012], 99.50th=[ 1106], 99.90th=[ 1319], 99.95th=[ 1352],
| 99.99th=[ 1352]
bw (KB/s): min= 264, max= 411, per=100.00%, avg=362.59, stdev=30.10
lat (msec): 20=0.07%, 50=5.85%, 100=12.72%, 250=27.59%, 500=24.50%
lat (msec): 750=19.12%, 1000=9.05%, 2000=1.09%
cpu: usr=0.09%, sys=0.41%, ctx=2751, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.3%, 16=0.6%, 32=98.9%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=2751/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=11004KB, aggrb=362KB/s, minb=362KB/s, maxb=362KB/s, mint=30385msec, maxt=30385msec

Disk stats (read/write):
sda: ios=2732/0, merge=0/0, ticks=953790/0, in_queue=963950, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [300K/0K /s] [75 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3817
read: io=9880.0KB, bw=335270 B/s, iops=81, runt= 30176msec
slat (usec): min=4, max=50, avg=17.58, stdev= 4.03
clat (usec): min=217, max=755355, avg=195368.11, stdev=148034.81
lat (usec): min=269, max=755375, avg=195386.26, stdev=148035.38
clat percentiles (usec):
| 1.00th=[ 1896], 5.00th=[30080], 10.00th=[45824], 20.00th=[71168],
| 30.00th=[95744], 40.00th=[120320], 50.00th=[150528], 60.00th=[187392],
| 70.00th=[246784], 80.00th=[313344], 90.00th=[423936], 95.00th=[501760],
| 99.00th=[602112], 99.50th=[643072], 99.90th=[700416], 99.95th=[724992],
| 99.99th=[757760]
bw (KB/s): min= 278, max= 1078, per=100.00%, avg=327.22, stdev=102.27
lat (usec): 250=0.04%, 500=0.04%, 750=0.16%, 1000=0.16%
lat (msec): 2=0.69%, 4=1.70%, 10=0.97%, 20=0.16%, 50=7.69%
lat (msec): 100=20.12%, 250=38.70%, 500=24.37%, 750=5.14%, 1000=0.04%
cpu: usr=0.18%, sys=0.25%, ctx=2470, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.2%, 8=0.3%, 16=99.4%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=2470/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=9880KB, aggrb=327KB/s, minb=327KB/s, maxb=327KB/s, mint=30176msec, maxt=30176msec

Disk stats (read/write):
sda: ios=2468/0, merge=0/0, ticks=480783/0, in_queue=481256, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [304K/0K /s] [76 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3821
read: io=8944.0KB, bw=304153 B/s, iops=74, runt= 30112msec
slat (usec): min=4, max=139, avg=18.18, stdev= 3.87
clat (msec): min=13, max=307, avg=107.66, stdev=48.10
lat (msec): min=13, max=307, avg=107.68, stdev=48.10
clat percentiles (msec):
| 1.00th=[ 29], 5.00th=[ 38], 10.00th=[ 47], 20.00th=[ 61],
| 30.00th=[ 75], 40.00th=[ 93], 50.00th=[ 108], 60.00th=[ 120],
| 70.00th=[ 135], 80.00th=[ 151], 90.00th=[ 169], 95.00th=[ 186],
| 99.00th=[ 243], 99.50th=[ 251], 99.90th=[ 289], 99.95th=[ 297],
| 99.99th=[ 310]
bw (KB/s): min= 264, max= 327, per=100.00%, avg=297.14, stdev=15.40
lat (msec): 20=0.04%, 50=11.99%, 100=33.23%, 250=54.16%, 500=0.58%
cpu: usr=0.10%, sys=0.31%, ctx=2236, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.2%, 8=99.7%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=2236/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=8944KB, aggrb=297KB/s, minb=297KB/s, maxb=297KB/s, mint=30112msec, maxt=30112msec

Disk stats (read/write):
sda: ios=2224/0, merge=0/0, ticks=238710/0, in_queue=239166, util=99.71%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [236K/0K /s] [59 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3825
read: io=6860.0KB, bw=233640 B/s, iops=57, runt= 30066msec
slat (usec): min=5, max=49, avg=18.10, stdev= 2.69
clat (msec): min=18, max=151, avg=70.06, stdev=19.49
lat (msec): min=18, max=151, avg=70.08, stdev=19.49
clat percentiles (msec):
| 1.00th=[ 35], 5.00th=[ 42], 10.00th=[ 47], 20.00th=[ 53],
| 30.00th=[ 59], 40.00th=[ 64], 50.00th=[ 70], 60.00th=[ 75],
| 70.00th=[ 80], 80.00th=[ 87], 90.00th=[ 95], 95.00th=[ 104],
| 99.00th=[ 124], 99.50th=[ 130], 99.90th=[ 151], 99.95th=[ 151],
| 99.99th=[ 151]
bw (KB/s): min= 201, max= 251, per=100.00%, avg=228.00, stdev=13.05
lat (msec): 20=0.06%, 50=15.57%, 100=77.73%, 250=6.65%
cpu: usr=0.10%, sys=0.21%, ctx=1715, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=99.8%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=1715/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=6860KB, aggrb=228KB/s, minb=228KB/s, maxb=228KB/s, mint=30066msec, maxt=30066msec

Disk stats (read/write):
sda: ios=1706/0, merge=0/0, ticks=119393/0, in_queue=119530, util=99.70%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [212K/0K /s] [53 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3829
read: io=6596.0KB, bw=224993 B/s, iops=54, runt= 30020msec
slat (usec): min=9, max=48, avg=22.16, stdev= 2.08
clat (msec): min=15, max=149, avg=36.37, stdev= 9.31
lat (msec): min=15, max=149, avg=36.40, stdev= 9.31
clat percentiles (msec):
| 1.00th=[ 19], 5.00th=[ 23], 10.00th=[ 26], 20.00th=[ 29],
| 30.00th=[ 32], 40.00th=[ 35], 50.00th=[ 37], 60.00th=[ 38],
| 70.00th=[ 41], 80.00th=[ 44], 90.00th=[ 48], 95.00th=[ 51],
| 99.00th=[ 58], 99.50th=[ 60], 99.90th=[ 145], 99.95th=[ 151],
| 99.99th=[ 151]
bw (KB/s): min= 167, max= 262, per=100.00%, avg=219.48, stdev=17.43
lat (msec): 20=1.76%, 50=92.12%, 100=6.00%, 250=0.12%
cpu: usr=0.08%, sys=0.23%, ctx=1650, majf=0, minf=0
IO depths: 1=0.1%, 2=99.9%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=1649/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=6596KB, aggrb=219KB/s, minb=219KB/s, maxb=219KB/s, mint=30020msec, maxt=30020msec

Disk stats (read/write):
sda: ios=1641/0, merge=0/0, ticks=59690/0, in_queue=59730, util=99.70%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [224K/0K /s] [56 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=3833
read: io=6656.0KB, bw=227153 B/s, iops=55, runt= 30005msec
slat (usec): min=12, max=145, avg=27.24, stdev= 3.99
clat (msec): min=3, max=35, avg=18.00, stdev= 5.54
lat (msec): min=3, max=35, avg=18.02, stdev= 5.54
clat percentiles (usec):
| 1.00th=[ 6752], 5.00th=[ 8896], 10.00th=[10816], 20.00th=[12992],
| 30.00th=[14656], 40.00th=[16192], 50.00th=[18048], 60.00th=[19584],
| 70.00th=[21120], 80.00th=[22912], 90.00th=[25472], 95.00th=[27264],
| 99.00th=[30592], 99.50th=[31104], 99.90th=[32384], 99.95th=[35584],
| 99.99th=[35584]
bw (KB/s): min= 182, max= 254, per=100.00%, avg=221.51, stdev=16.49
lat (msec): 4=0.06%, 10=7.57%, 20=55.05%, 50=37.32%
cpu: usr=0.10%, sys=0.16%, ctx=1664, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=1664/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=6656KB, aggrb=221KB/s, minb=221KB/s, maxb=221KB/s, mint=30005msec, maxt=30005msec

Disk stats (read/write):
sda: ios=1657/0, merge=0/0, ticks=29793/0, in_queue=29800, util=99.34%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1024
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [0.0% done] [0K/132K /s] [0 /33 iops] [eta 11d:09h:04m:56s]
writetest: (groupid=0, jobs=1): err= 0: pid=3838
write: io=13948KB, bw=454850 B/s, iops=111, runt= 31401msec
slat (usec): min=3, max=878364, avg=8661.37, stdev=51377.60
clat (msec): min=1188, max=12599, avg=8344.81, stdev=2246.20
lat (msec): min=1426, max=12599, avg=8353.47, stdev=2245.59
clat percentiles (msec):
| 1.00th=[ 1827], 5.00th=[ 2933], 10.00th=[ 4555], 20.00th=[ 7570],
| 30.00th=[ 8160], 40.00th=[ 8455], 50.00th=[ 8717], 60.00th=[ 9110],
| 70.00th=[ 9503], 80.00th=[10028], 90.00th=[10683], 95.00th=[11076],
| 99.00th=[11600], 99.50th=[11731], 99.90th=[12256], 99.95th=[12387],
| 99.99th=[12649]
bw (KB/s): min= 0, max= 815, per=94.25%, avg=418.49, stdev=125.16
lat (msec): 2000=1.86%, >=2000=98.14%
cpu: usr=0.01%, sys=0.08%, ctx=235, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.5%, 32=0.9%, >=64=98.2%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
issued: total=r=0/w=0/d=3487, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=13948KB, aggrb=444KB/s, minb=444KB/s, maxb=444KB/s, mint=31401msec, maxt=31401msec

Disk stats (read/write):
sda: ios=85/3455, merge=0/0, ticks=3414/4329547, in_queue=4392543, util=100.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=32
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/428K /s] [0 /107 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3861
write: io=12968KB, bw=438475 B/s, iops=107, runt= 30285msec
slat (usec): min=3, max=138, avg=19.15, stdev= 3.63
clat (usec): min=419, max=1160.5K, avg=298800.84, stdev=230187.03
lat (usec): min=471, max=1160.6K, avg=298820.55, stdev=230187.44
clat percentiles (msec):
| 1.00th=[ 11], 5.00th=[ 32], 10.00th=[ 52], 20.00th=[ 86],
| 30.00th=[ 126], 40.00th=[ 176], 50.00th=[ 231], 60.00th=[ 310],
| 70.00th=[ 408], 80.00th=[ 519], 90.00th=[ 652], 95.00th=[ 734],
| 99.00th=[ 873], 99.50th=[ 930], 99.90th=[ 1057], 99.95th=[ 1123],
| 99.99th=[ 1156]
bw (KB/s): min= 337, max= 881, per=100.00%, avg=428.34, stdev=73.09
lat (usec): 500=0.03%, 1000=0.03%
lat (msec): 2=0.09%, 4=0.15%, 10=0.68%, 20=1.30%, 50=7.25%
lat (msec): 100=14.53%, 250=28.16%, 500=25.72%, 750=17.46%, 1000=4.32%
lat (msec): 2000=0.28%
cpu: usr=0.22%, sys=0.38%, ctx=3242, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=0.5%, 32=99.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3242, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=12968KB, aggrb=428KB/s, minb=428KB/s, maxb=428KB/s, mint=30285msec, maxt=30285msec

Disk stats (read/write):
sda: ios=84/3228, merge=0/0, ticks=24/959137, in_queue=963340, util=99.78%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=16
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/428K /s] [0 /107 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3883
write: io=13024KB, bw=442678 B/s, iops=108, runt= 30127msec
slat (usec): min=3, max=47, avg=19.10, stdev= 2.97
clat (usec): min=434, max=622395, avg=147974.19, stdev=107052.31
lat (usec): min=484, max=622416, avg=147993.84, stdev=107052.73
clat percentiles (msec):
| 1.00th=[ 8], 5.00th=[ 26], 10.00th=[ 37], 20.00th=[ 58],
| 30.00th=[ 77], 40.00th=[ 98], 50.00th=[ 121], 60.00th=[ 145],
| 70.00th=[ 182], 80.00th=[ 231], 90.00th=[ 306], 95.00th=[ 367],
| 99.00th=[ 474], 99.50th=[ 498], 99.90th=[ 586], 99.95th=[ 619],
| 99.99th=[ 619]
bw (KB/s): min= 333, max= 948, per=100.00%, avg=432.32, stdev=76.93
lat (usec): 500=0.03%, 1000=0.03%
lat (msec): 2=0.18%, 4=0.31%, 10=1.04%, 20=1.72%, 50=13.14%
lat (msec): 100=24.29%, 250=42.29%, 500=16.46%, 750=0.49%
cpu: usr=0.15%, sys=0.45%, ctx=3258, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=0.2%, 16=99.5%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3256, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=13024KB, aggrb=432KB/s, minb=432KB/s, maxb=432KB/s, mint=30127msec, maxt=30127msec

Disk stats (read/write):
sda: ios=84/3231, merge=0/0, ticks=17/476620, in_queue=478430, util=99.77%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=8
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/416K /s] [0 /104 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3906
write: io=12484KB, bw=424831 B/s, iops=103, runt= 30091msec
slat (usec): min=3, max=46, avg=19.05, stdev= 2.84
clat (usec): min=388, max=290503, avg=77074.76, stdev=39278.20
lat (usec): min=436, max=290523, avg=77094.38, stdev=39278.62
clat percentiles (msec):
| 1.00th=[ 5], 5.00th=[ 23], 10.00th=[ 30], 20.00th=[ 41],
| 30.00th=[ 51], 40.00th=[ 63], 50.00th=[ 74], 60.00th=[ 86],
| 70.00th=[ 97], 80.00th=[ 112], 90.00th=[ 131], 95.00th=[ 145],
| 99.00th=[ 182], 99.50th=[ 202], 99.90th=[ 225], 99.95th=[ 245],
| 99.99th=[ 289]
bw (KB/s): min= 332, max= 952, per=100.00%, avg=415.36, stdev=80.17
lat (usec): 500=0.03%, 750=0.03%
lat (msec): 2=0.45%, 4=0.42%, 10=0.87%, 20=2.47%, 50=24.64%
lat (msec): 100=43.10%, 250=27.97%, 500=0.03%
cpu: usr=0.14%, sys=0.44%, ctx=3121, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=0.1%, 8=99.8%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3121, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=12484KB, aggrb=414KB/s, minb=414KB/s, maxb=414KB/s, mint=30091msec, maxt=30091msec

Disk stats (read/write):
sda: ios=84/3106, merge=0/0, ticks=20/238780, in_queue=239160, util=99.78%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=4
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/436K /s] [0 /109 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3928
write: io=12872KB, bw=438823 B/s, iops=107, runt= 30037msec
slat (usec): min=4, max=49, avg=19.17, stdev= 2.65
clat (usec): min=369, max=102682, avg=37297.06, stdev=15596.69
lat (usec): min=418, max=102702, avg=37316.80, stdev=15597.26
clat percentiles (msec):
| 1.00th=[ 3], 5.00th=[ 16], 10.00th=[ 20], 20.00th=[ 25],
| 30.00th=[ 28], 40.00th=[ 32], 50.00th=[ 36], 60.00th=[ 41],
| 70.00th=[ 45], 80.00th=[ 50], 90.00th=[ 58], 95.00th=[ 66],
| 99.00th=[ 80], 99.50th=[ 88], 99.90th=[ 97], 99.95th=[ 99],
| 99.99th=[ 103]
bw (KB/s): min= 309, max= 955, per=100.00%, avg=428.51, stdev=82.23
lat (usec): 500=0.03%, 750=0.03%
lat (msec): 2=0.75%, 4=0.71%, 10=0.84%, 20=8.70%, 50=68.92%
lat (msec): 100=19.98%, 250=0.03%
cpu: usr=0.19%, sys=0.41%, ctx=3218, majf=0, minf=0
IO depths: 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3218, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=12872KB, aggrb=428KB/s, minb=428KB/s, maxb=428KB/s, mint=30037msec, maxt=30037msec

Disk stats (read/write):
sda: ios=84/3206, merge=0/0, ticks=20/119400, in_queue=119513, util=99.78%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/416K /s] [0 /104 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3950
write: io=12812KB, bw=436981 B/s, iops=106, runt= 30023msec
slat (usec): min=5, max=155, avg=22.97, stdev= 3.82
clat (usec): min=393, max=73421, avg=18715.34, stdev=9080.87
lat (usec): min=441, max=73443, avg=18738.87, stdev=9081.44
clat percentiles (usec):
| 1.00th=[ 1048], 5.00th=[ 5280], 10.00th=[ 7968], 20.00th=[11200],
| 30.00th=[12992], 40.00th=[15424], 50.00th=[18048], 60.00th=[20096],
| 70.00th=[22656], 80.00th=[26496], 90.00th=[30848], 95.00th=[35072],
| 99.00th=[42752], 99.50th=[44288], 99.90th=[51456], 99.95th=[73216],
| 99.99th=[73216]
bw (KB/s): min= 317, max= 922, per=100.00%, avg=426.53, stdev=73.58
lat (usec): 500=0.03%, 750=0.06%, 1000=0.72%
lat (msec): 2=1.03%, 4=0.59%, 10=12.55%, 20=44.21%, 50=40.65%
lat (msec): 100=0.16%
cpu: usr=0.20%, sys=0.40%, ctx=3203, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3203, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=12812KB, aggrb=426KB/s, minb=426KB/s, maxb=426KB/s, mint=30023msec, maxt=30023msec

Disk stats (read/write):
sda: ios=84/3189, merge=0/0, ticks=17/59607, in_queue=59646, util=99.76%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [w] [100.0% done] [0K/416K /s] [0 /104 iops] [eta 00m:00s]
writetest: (groupid=0, jobs=1): err= 0: pid=3972
write: io=12616KB, bw=430597 B/s, iops=105, runt= 30002msec
slat (usec): min=6, max=48, avg=27.49, stdev= 3.66
clat (usec): min=342, max=35159, avg=9477.07, stdev=7171.91
lat (usec): min=358, max=35173, avg=9505.14, stdev=7172.29
clat percentiles (usec):
| 1.00th=[ 470], 5.00th=[ 564], 10.00th=[ 660], 20.00th=[ 820],
| 30.00th=[ 4576], 40.00th=[ 5920], 50.00th=[ 7776], 60.00th=[11968],
| 70.00th=[13760], 80.00th=[15424], 90.00th=[18816], 95.00th=[22912],
| 99.00th=[27008], 99.50th=[30080], 99.90th=[35072], 99.95th=[35072],
| 99.99th=[35072]
bw (KB/s): min= 319, max= 915, per=100.00%, avg=420.32, stdev=76.45
lat (usec): 500=2.09%, 750=16.52%, 1000=1.71%
lat (msec): 2=0.19%, 4=6.21%, 10=27.01%, 20=37.86%, 50=8.40%
cpu: usr=0.13%, sys=0.36%, ctx=3155, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=3154, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=12616KB, aggrb=420KB/s, minb=420KB/s, maxb=420KB/s, mint=30002msec, maxt=30002msec

Disk stats (read/write):
sda: ios=84/3141, merge=0/0, ticks=20/29754, in_queue=29787, util=99.29%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=1
2.0.8
Starting 1 process
Jobs: 1 (f=1): [r] [100.0% done] [32K/0K /s] [8 /0 iops] [eta 00m:00s]
readtest: (groupid=0, jobs=1): err= 0: pid=4518
read: io=712704 B, bw=23655 B/s, iops=5, runt= 30128msec
slat (usec): min=13, max=25, avg=14.97, stdev= 2.39
clat (msec): min=48, max=3651, avg=172.92, stdev=269.66
lat (msec): min=48, max=3651, avg=172.94, stdev=269.66
clat percentiles (msec):
| 1.00th=[ 52], 5.00th=[ 82], 10.00th=[ 92], 20.00th=[ 109],
| 30.00th=[ 121], 40.00th=[ 137], 50.00th=[ 151], 60.00th=[ 163],
| 70.00th=[ 180], 80.00th=[ 202], 90.00th=[ 223], 95.00th=[ 243],
| 99.00th=[ 269], 99.50th=[ 3654], 99.90th=[ 3654], 99.95th=[ 3654],
| 99.99th=[ 3654]
bw (KB/s): min= 1, max= 40, per=100.00%, avg=25.22, stdev= 5.62
lat (msec): 50=0.57%, 100=13.79%, 250=82.18%, 500=2.87%, >=2000=0.57%
cpu: usr=0.00%, sys=0.03%, ctx=182, majf=0, minf=0
IO depths: 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=174/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=696KB, aggrb=23KB/s, minb=23KB/s, maxb=23KB/s, mint=30128msec, maxt=30128msec

Disk stats (read/write):
sr0: ios=172/0, merge=0/0, ticks=29810/0, in_queue=29834, util=99.57%

root@host:~# fio ./read.ini
readtest: (g=0): rw=randread, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process

readtest: (groupid=0, jobs=1): err= 0: pid=4262
read: io=97140KB, bw=852105KB/s, iops=213026, runt= 114msec
slat (usec): min=2, max=118, avg= 2.47, stdev= 0.92
clat (usec): min=3, max=121, avg= 5.51, stdev= 0.94
lat (usec): min=6, max=124, avg= 8.34, stdev= 1.21
clat percentiles (usec):
| 1.00th=[ 5], 5.00th=[ 5], 10.00th=[ 5], 20.00th=[ 5],
| 30.00th=[ 5], 40.00th=[ 5], 50.00th=[ 6], 60.00th=[ 6],
| 70.00th=[ 6], 80.00th=[ 6], 90.00th=[ 6], 95.00th=[ 6],
| 99.00th=[ 6], 99.50th=[ 6], 99.90th=[ 9], 99.95th=[ 13],
| 99.99th=[ 15]
lat (usec): 4=0.01%, 10=99.91%, 20=0.07%, 50=0.01%, 250=0.01%
cpu: usr=34.51%, sys=64.60%, ctx=1, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=24285/w=0/d=0, short=r=0/w=0/d=0

Run status group 0 (all jobs):
READ: io=97140KB, aggrb=852105KB/s, minb=852105KB/s, maxb=852105KB/s, mint=114msec, maxt=114msec

Disk stats (read/write):
ram1: ios=0/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%

root@host:~# fio ./write.ini
writetest: (g=0): rw=randwrite, bs=4K-4K/4K-4K, ioengine=libaio, iodepth=2
2.0.8
Starting 1 process

writetest: (groupid=0, jobs=1): err= 0: pid=4293
write: io=97140KB, bw=693857KB/s, iops=173464, runt= 140msec
slat (usec): min=3, max=120, avg= 3.35, stdev= 0.96
clat (usec): min=4, max=124, avg= 6.44, stdev= 1.25
lat (usec): min=9, max=128, avg=10.12, stdev= 1.50
clat percentiles (usec):
| 1.00th=[ 6], 5.00th=[ 6], 10.00th=[ 6], 20.00th=[ 6],
| 30.00th=[ 6], 40.00th=[ 6], 50.00th=[ 6], 60.00th=[ 7],
| 70.00th=[ 7], 80.00th=[ 7], 90.00th=[ 7], 95.00th=[ 7],
| 99.00th=[ 8], 99.50th=[ 8], 99.90th=[ 10], 99.95th=[ 14],
| 99.99th=[ 52]
lat (usec): 10=99.80%, 20=0.19%, 100=0.01%, 250=0.01%
cpu: usr=20.86%, sys=76.26%, ctx=1, majf=0, minf=0
IO depths: 1=0.1%, 2=100.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete: 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued: total=r=0/w=0/d=24285, short=r=0/w=0/d=0

Run status group 0 (all jobs):
WRITE: io=97140KB, aggrb=693857KB/s, minb=693857KB/s, maxb=693857KB/s, mint=140msec, maxt=140msec

Disk stats (read/write):
ram1: ios=0/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%