Proxmox VE Ceph Benchmark 2023/12 - Fast SSDs and network speeds in a Proxmox VE Ceph Reef cluster

[kellogs]

root@stor-16-132:~# rados bench -p cephNVME 10 write --no-cleanup
hints = 1
Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
Object prefix: benchmark_data_stor-16-132_393342
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
1 16 1534 1518 6071.1 6072 0.00901612 0.0104816
2 16 3106 3090 6179.18 6288 0.00788136 0.0103317
3 16 4698 4682 6241.87 6368 0.00766676 0.0102388
4 16 6277 6261 6260.24 6316 0.01143 0.0102077
5 16 7867 7851 6280.05 6360 0.00852833 0.0101792
6 16 9432 9416 6276.59 6260 0.00740382 0.0101867
7 16 11012 10996 6282.69 6320 0.00780489 0.0101788
8 16 12563 12547 6272.76 6204 0.00832499 0.010194
9 16 14136 14120 6274.8 6292 0.0109357 0.0101933
10 15 15726 15711 6283.66 6364 0.0125442 0.0101773
Total time run: 10.0076
Total writes made: 15726
Write size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 6285.62
Stddev Bandwidth: 90.4006
Max bandwidth (MB/sec): 6368
Min bandwidth (MB/sec): 6072
Average IOPS: 1571
Stddev IOPS: 22.6001
Max IOPS: 1592
Min IOPS: 1518
Average Latency(s): 0.0101771
Stddev Latency(s): 0.00256609
Max latency(s): 0.0379702
Min latency(s): 0.00554199

 

[kellogs]

root@stor-16-132:~# rados bench -p cephNVME 10 seq
hints = 1
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
1 16 1293 1277 5107 5108 0.0060601 0.0120894
2 16 2807 2791 5581.14 6056 0.00590438 0.0110674
3 16 4263 4247 5661.77 5824 0.00580642 0.0109099
4 16 5683 5667 5665.95 5680 0.0172076 0.0109233
5 16 7119 7103 5681.44 5744 0.00726775 0.0109005
6 16 8554 8538 5691.02 5740 0.005632 0.010884
7 16 9928 9912 5662.97 5496 0.00798654 0.0109374
8 16 11314 11298 5648.02 5544 0.00485397 0.0109623
9 16 12708 12692 5639.77 5576 0.00656022 0.0109773
10 13 14081 14068 5626.14 5504 0.0156334 0.0110054
Total time run: 10.0142
Total reads made: 14081
Read size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 5624.4
Average IOPS: 1406
Stddev IOPS: 62.5421
Max IOPS: 1514
Min IOPS: 1277
Average Latency(s): 0.0110075
Max latency(s): 0.119927
Min latency(s): 0.00303387

 

[kellogs]

root@stor-16-132:~# rados bench -p cephNVME 10 rand
hints = 1
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
1 15 1137 1122 4487.23 4488 0.00858927 0.0137572
2 16 2430 2414 4827.38 5168 0.0552671 0.0127482
3 16 3717 3701 4934.12 5148 0.00452533 0.0125491
4 16 5096 5080 5079.47 5516 0.0198027 0.0122057
5 16 6519 6503 5201.88 5692 0.00786024 0.0119145
6 15 7891 7876 5249.97 5492 0.00689363 0.0118018
7 16 9222 9206 5259.91 5320 0.00683547 0.0117773
8 16 10564 10548 5273.28 5368 0.0182627 0.011749
9 15 11952 11937 5304.47 5556 0.0100541 0.0116794
10 13 13421 13408 5362.28 5884 0.0379289 0.0115557
Total time run: 10.0158
Total reads made: 13421
Read size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 5359.95
Average IOPS: 1339
Stddev IOPS: 95.3331
Max IOPS: 1471
Min IOPS: 1122
Average Latency(s): 0.0115585
Max latency(s): 0.14316
Min latency(s): 0.00216847

 

[kellogs]

root@stor-16-132:~# rados bench -p cephNVME 10 write -t 4 --run-name client1
hints = 1
Maintaining 4 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 10 seconds or 0 objects
Object prefix: benchmark_data_stor-16-132_398367
sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s)
0 0 0 0 0 0 - 0
1 4 554 550 2199.85 2200 0.00522261 0.00724648
2 4 1133 1129 2257.8 2316 0.00557932 0.00707488
3 4 1721 1717 2289.11 2352 0.00592636 0.00697536
4 4 2306 2302 2301.76 2340 0.00518888 0.00694495
5 4 2891 2887 2309.37 2340 0.00713633 0.00692432
6 4 3480 3476 2317.1 2356 0.00562834 0.00689941
7 4 4071 4067 2323.77 2364 0.0071261 0.00688114
8 4 4664 4660 2329.77 2372 0.00621837 0.00686399
9 4 5249 5245 2330.88 2340 0.00735179 0.00686155
10 4 5831 5827 2330.57 2328 0.0059157 0.00686127
Total time run: 10.0049
Total writes made: 5831
Write size: 4194304
Object size: 4194304
Bandwidth (MB/sec): 2331.26
Stddev Bandwidth: 48.8462
Max bandwidth (MB/sec): 2372
Min bandwidth (MB/sec): 2200
Average IOPS: 582
Stddev IOPS: 12.2116
Max IOPS: 593
Min IOPS: 550
Average Latency(s): 0.00686079
Stddev Latency(s): 0.00149619
Max latency(s): 0.0207001
Min latency(s): 0.00450528
Cleaning up (deleting benchmark objects)
Removed 5831 objects
Clean up completed and total clean up time :0.79139