Hi All,
I've a ceph Cluster with 3 nodes HPE each with 10xSAS 1TB and 2xnvme 1TB below the config.
The replica and ceph network is 10Gb but the performance are very low...
in VM I got (in sequential mode) Read: 230MBps Write: 65MBps
What I can do/check to tune my storage environment?
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
device 2 osd.2 class hdd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class hdd
device 8 osd.8 class hdd
device 9 osd.9 class hdd
device 10 osd.10 class hdd
device 11 osd.11 class hdd
device 12 osd.12 class hdd
device 13 osd.13 class hdd
device 14 osd.14 class hdd
device 15 osd.15 class hdd
device 16 osd.16 class hdd
device 17 osd.17 class hdd
device 18 osd.18 class hdd
device 19 osd.19 class hdd
device 20 osd.20 class hdd
device 21 osd.21 class hdd
device 22 osd.22 class hdd
device 23 osd.23 class hdd
device 24 osd.24 class hdd
device 25 osd.25 class hdd
device 26 osd.26 class hdd
device 27 osd.27 class hdd
device 28 osd.28 class hdd
device 29 osd.29 class hdd
device 30 osd.30 class ssd
device 31 osd.31 class ssd
device 32 osd.32 class ssd
device 33 osd.33 class ssd
device 34 osd.34 class ssd
device 35 osd.35 class ssd
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root
# buckets
host HCI11 {
id -3 # do not change unnecessarily
id -4 class hdd # do not change unnecessarily
id -9 class ssd # do not change unnecessarily
# weight 10.832
alg straw2
hash 0 # rjenkins1
item osd.0 weight 0.895
item osd.1 weight 0.895
item osd.2 weight 0.895
item osd.3 weight 0.895
item osd.4 weight 0.895
item osd.5 weight 0.910
item osd.6 weight 0.895
item osd.7 weight 0.895
item osd.8 weight 0.895
item osd.9 weight 0.895
item osd.30 weight 0.931
item osd.31 weight 0.931
}
host HCI12 {
id -5 # do not change unnecessarily
id -6 class hdd # do not change unnecessarily
id -10 class ssd # do not change unnecessarily
# weight 10.818
alg straw2
hash 0 # rjenkins1
item osd.10 weight 0.895
item osd.11 weight 0.895
item osd.12 weight 0.895
item osd.13 weight 0.895
item osd.14 weight 0.895
item osd.15 weight 0.895
item osd.16 weight 0.895
item osd.17 weight 0.895
item osd.18 weight 0.895
item osd.19 weight 0.895
item osd.32 weight 0.931
item osd.33 weight 0.931
}
host HCI13 {
id -7 # do not change unnecessarily
id -8 class hdd # do not change unnecessarily
id -11 class ssd # do not change unnecessarily
# weight 10.818
alg straw2
hash 0 # rjenkins1
item osd.20 weight 0.895
item osd.21 weight 0.895
item osd.22 weight 0.895
item osd.23 weight 0.895
item osd.24 weight 0.895
item osd.25 weight 0.895
item osd.26 weight 0.895
item osd.27 weight 0.895
item osd.28 weight 0.895
item osd.29 weight 0.895
item osd.34 weight 0.931
item osd.35 weight 0.931
}
root default {
id -1 # do not change unnecessarily
id -2 class hdd # do not change unnecessarily
id -12 class ssd # do not change unnecessarily
# weight 32.468
alg straw2
hash 0 # rjenkins1
item HCI11 weight 10.832
item HCI12 weight 10.818
item HCI13 weight 10.818
}
# rules
rule replicated_rule {
id 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
# end crush map
I've a ceph Cluster with 3 nodes HPE each with 10xSAS 1TB and 2xnvme 1TB below the config.
The replica and ceph network is 10Gb but the performance are very low...
in VM I got (in sequential mode) Read: 230MBps Write: 65MBps
What I can do/check to tune my storage environment?
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
device 2 osd.2 class hdd
device 3 osd.3 class hdd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class hdd
device 8 osd.8 class hdd
device 9 osd.9 class hdd
device 10 osd.10 class hdd
device 11 osd.11 class hdd
device 12 osd.12 class hdd
device 13 osd.13 class hdd
device 14 osd.14 class hdd
device 15 osd.15 class hdd
device 16 osd.16 class hdd
device 17 osd.17 class hdd
device 18 osd.18 class hdd
device 19 osd.19 class hdd
device 20 osd.20 class hdd
device 21 osd.21 class hdd
device 22 osd.22 class hdd
device 23 osd.23 class hdd
device 24 osd.24 class hdd
device 25 osd.25 class hdd
device 26 osd.26 class hdd
device 27 osd.27 class hdd
device 28 osd.28 class hdd
device 29 osd.29 class hdd
device 30 osd.30 class ssd
device 31 osd.31 class ssd
device 32 osd.32 class ssd
device 33 osd.33 class ssd
device 34 osd.34 class ssd
device 35 osd.35 class ssd
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root
# buckets
host HCI11 {
id -3 # do not change unnecessarily
id -4 class hdd # do not change unnecessarily
id -9 class ssd # do not change unnecessarily
# weight 10.832
alg straw2
hash 0 # rjenkins1
item osd.0 weight 0.895
item osd.1 weight 0.895
item osd.2 weight 0.895
item osd.3 weight 0.895
item osd.4 weight 0.895
item osd.5 weight 0.910
item osd.6 weight 0.895
item osd.7 weight 0.895
item osd.8 weight 0.895
item osd.9 weight 0.895
item osd.30 weight 0.931
item osd.31 weight 0.931
}
host HCI12 {
id -5 # do not change unnecessarily
id -6 class hdd # do not change unnecessarily
id -10 class ssd # do not change unnecessarily
# weight 10.818
alg straw2
hash 0 # rjenkins1
item osd.10 weight 0.895
item osd.11 weight 0.895
item osd.12 weight 0.895
item osd.13 weight 0.895
item osd.14 weight 0.895
item osd.15 weight 0.895
item osd.16 weight 0.895
item osd.17 weight 0.895
item osd.18 weight 0.895
item osd.19 weight 0.895
item osd.32 weight 0.931
item osd.33 weight 0.931
}
host HCI13 {
id -7 # do not change unnecessarily
id -8 class hdd # do not change unnecessarily
id -11 class ssd # do not change unnecessarily
# weight 10.818
alg straw2
hash 0 # rjenkins1
item osd.20 weight 0.895
item osd.21 weight 0.895
item osd.22 weight 0.895
item osd.23 weight 0.895
item osd.24 weight 0.895
item osd.25 weight 0.895
item osd.26 weight 0.895
item osd.27 weight 0.895
item osd.28 weight 0.895
item osd.29 weight 0.895
item osd.34 weight 0.931
item osd.35 weight 0.931
}
root default {
id -1 # do not change unnecessarily
id -2 class hdd # do not change unnecessarily
id -12 class ssd # do not change unnecessarily
# weight 32.468
alg straw2
hash 0 # rjenkins1
item HCI11 weight 10.832
item HCI12 weight 10.818
item HCI13 weight 10.818
}
# rules
rule replicated_rule {
id 0
type replicated
min_size 1
max_size 10
step take default
step chooseleaf firstn 0 type host
step emit
}
# end crush map