I got 'HEALTH_WARN: 20 slow ops, oldest one blocked for 84 sec, daemons [osd.57,osd.58,osd.59,osd.66,osd.67,osd.68,osd.77,osd.79] have slow ops.'
However, the latency of OSD is very low. just 2ms
What should I check?
However, the latency of OSD is very low. just 2ms
What should I check?
Code:
# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54
# devices
device 11 osd.11 class clay
device 12 osd.12 class ssd
device 13 osd.13 class clay
device 14 osd.14 class ssd
device 15 osd.15 class clay
device 16 osd.16 class clay
device 21 osd.21 class clay
device 22 osd.22 class ssd
device 23 osd.23 class clay
device 24 osd.24 class ssd
device 25 osd.25 class clay
device 26 osd.26 class clay
device 31 osd.31 class clay
device 32 osd.32 class ssd
device 33 osd.33 class clay
device 34 osd.34 class ssd
device 35 osd.35 class clay
device 36 osd.36 class clay
device 55 osd.55 class clay
device 56 osd.56 class destroy
device 57 osd.57 class clay
device 58 osd.58 class destroy
device 59 osd.59 class clay
device 64 osd.64 class clay
device 65 osd.65 class destroy
device 66 osd.66 class clay
device 67 osd.67 class clay
device 68 osd.68 class clay
device 76 osd.76 class clay
device 77 osd.77 class clay
device 78 osd.78 class destroy
device 79 osd.79 class clay
# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root
# buckets
host pve1 {
id -3 # do not change unnecessarily
id -32 class clay # do not change unnecessarily
id -13 class ssd # do not change unnecessarily
id -15 class destroy # do not change unnecessarily
# weight 1.84119
alg straw2
hash 0 # rjenkins1
item osd.12 weight 0.93149
item osd.14 weight 0.90970
}
host pve2 {
id -5 # do not change unnecessarily
id -33 class clay # do not change unnecessarily
id -14 class ssd # do not change unnecessarily
id -17 class destroy # do not change unnecessarily
# weight 1.86298
alg straw2
hash 0 # rjenkins1
item osd.22 weight 0.93149
item osd.24 weight 0.93149
}
host pve3 {
id -6 # do not change unnecessarily
id -34 class clay # do not change unnecessarily
id -7 class ssd # do not change unnecessarily
id -18 class destroy # do not change unnecessarily
# weight 1.86298
alg straw2
hash 0 # rjenkins1
item osd.32 weight 0.93149
item osd.34 weight 0.93149
}
pod system {
id -2 # do not change unnecessarily
id -66 class clay # do not change unnecessarily
id -65 class ssd # do not change unnecessarily
id -19 class destroy # do not change unnecessarily
# weight 5.56714
alg straw2
hash 0 # rjenkins1
item pve1 weight 1.84119
item pve2 weight 1.86298
item pve3 weight 1.86298
}
host hec11-pve1 {
id -28 # do not change unnecessarily
id -64 class clay # do not change unnecessarily
id -63 class ssd # do not change unnecessarily
id -20 class destroy # do not change unnecessarily
# weight 7.36475
alg straw2
hash 0 # rjenkins1
item osd.13 weight 1.86298
item osd.11 weight 1.81940
item osd.15 weight 1.86298
item osd.16 weight 1.81940
}
host hec11-pve2 {
id -30 # do not change unnecessarily
id -59 class clay # do not change unnecessarily
id -58 class ssd # do not change unnecessarily
id -21 class destroy # do not change unnecessarily
# weight 7.45190
alg straw2
hash 0 # rjenkins1
item osd.23 weight 1.86298
item osd.21 weight 1.86298
item osd.25 weight 1.86298
item osd.26 weight 1.86298
}
host hec11-pve3 {
id -35 # do not change unnecessarily
id -56 class clay # do not change unnecessarily
id -55 class ssd # do not change unnecessarily
id -22 class destroy # do not change unnecessarily
# weight 7.32117
alg straw2
hash 0 # rjenkins1
item osd.31 weight 1.81940
item osd.33 weight 1.81940
item osd.36 weight 1.81940
item osd.35 weight 1.86298
}
zone hot-ec11-clay566 {
id -11 # do not change unnecessarily
id -69 class clay # do not change unnecessarily
id -68 class ssd # do not change unnecessarily
id -23 class destroy # do not change unnecessarily
# weight 22.13783
alg straw2
hash 0 # rjenkins1
item hec11-pve1 weight 7.36475
item hec11-pve2 weight 7.45192
item hec11-pve3 weight 7.32117
}
zone standard-hot {
id -9 # do not change unnecessarily
id -75 class clay # do not change unnecessarily
id -74 class ssd # do not change unnecessarily
id -24 class destroy # do not change unnecessarily
# weight 22.13788
alg straw2
hash 0 # rjenkins1
item hot-ec11-clay566 weight 22.13788
}
host cec9-pve1 {
id -77 # do not change unnecessarily
id -87 class clay # do not change unnecessarily
id -86 class ssd # do not change unnecessarily
id -25 class destroy # do not change unnecessarily
# weight 9.18626
alg straw2
hash 0 # rjenkins1
item osd.57 weight 2.77379
item osd.59 weight 2.77379
item osd.55 weight 3.63869
}
host cec9-pve2 {
id -78 # do not change unnecessarily
id -84 class clay # do not change unnecessarily
id -83 class ssd # do not change unnecessarily
id -26 class destroy # do not change unnecessarily
# weight 13.86719
alg straw2
hash 0 # rjenkins1
item osd.64 weight 2.77379
item osd.66 weight 3.69780
item osd.67 weight 3.69780
item osd.68 weight 3.69780
}
host cec9-pve3 {
id -79 # do not change unnecessarily
id -81 class clay # do not change unnecessarily
id -80 class ssd # do not change unnecessarily
id -27 class destroy # do not change unnecessarily
# weight 10.16939
alg straw2
hash 0 # rjenkins1
item osd.76 weight 3.69780
item osd.77 weight 3.69780
item osd.79 weight 2.77379
}
zone cold-ec9-clay455 {
id -44 # do not change unnecessarily
id -50 class clay # do not change unnecessarily
id -47 class ssd # do not change unnecessarily
id -36 class destroy # do not change unnecessarily
# weight 33.22282
alg straw2
hash 0 # rjenkins1
item cec9-pve1 weight 9.18625
item cec9-pve2 weight 13.86719
item cec9-pve3 weight 10.16939
}
zone archive-cold {
id -10 # do not change unnecessarily
id -72 class clay # do not change unnecessarily
id -71 class ssd # do not change unnecessarily
id -37 class destroy # do not change unnecessarily
# weight 33.22284
alg straw2
hash 0 # rjenkins1
item cold-ec9-clay455 weight 33.22284
}
pod user {
id -4 # do not change unnecessarily
id -62 class clay # do not change unnecessarily
id -61 class ssd # do not change unnecessarily
id -38 class destroy # do not change unnecessarily
# weight 55.36072
alg straw2
hash 0 # rjenkins1
item standard-hot weight 22.13788
item archive-cold weight 33.22284
}
root default {
id -1 # do not change unnecessarily
id -43 class clay # do not change unnecessarily
id -16 class ssd # do not change unnecessarily
id -39 class destroy # do not change unnecessarily
# weight 60.92786
alg straw2
hash 0 # rjenkins1
item system weight 5.56714
item user weight 55.36072
}
root destroy {
id -8 # do not change unnecessarily
id -31 class clay # do not change unnecessarily
id -29 class ssd # do not change unnecessarily
id -12 class destroy # do not change unnecessarily
# weight 11.09515
alg straw2
hash 0 # rjenkins1
item osd.56 weight 2.77379
item osd.58 weight 2.77379
item osd.65 weight 2.77379
item osd.78 weight 2.77379
}
# rules
rule replicated_rule {
id 0
type replicated
step take default
step chooseleaf firstn 0 type host
step emit
}
rule replicated_user_hdd {
id 2
type replicated
step take cold-ec9-clay455
step chooseleaf firstn 0 type host
step emit
}
rule replicated_system_ssd {
id 3
type replicated
step take system class ssd
step chooseleaf firstn 0 type host
step emit
}
rule replicated_user_ssd {
id 4
type replicated
step take standard-hot class clay
step chooseleaf firstn 0 type host
step emit
}
rule autoscaler_ssd {
id 6
type replicated
step take system class clay
step chooseleaf firstn 0 type host
step emit
}
rule autoscaler_clay {
id 7
type replicated
step take user class clay
step chooseleaf firstn 0 type host
step emit
}
rule cephfs-userdata-tier_hec11-clay566-2025-0310 {
id 8
type erasure
step set_chooseleaf_tries 5
step set_choose_tries 100
step take hot-ec11-clay566 class clay
step choose indep 0 type osd
step emit
}
rule cephfs-userdata-tier_cec9-clay455-2025-0310 {
id 9
type erasure
step set_chooseleaf_tries 5
step set_choose_tries 100
step take cold-ec9-clay455 class clay
step choose indep 0 type osd
step emit
}
# end crush map
