Hello.
I have a Proxmox+Ceph cluster, 5 nodes (8 OSD per nodes).
This morning, I updated (to last 8.4) and rebooted each node (one at a time, moving the VM beforehand, etc).
I ended up with two "out" and "outdated" OSD (both "out" and "outdated"), one on node 3 and one on node 5.
Tried to restart OSD-34 (one of the two "out" OSD, this one is on node 5) manually, didn't work.
A quick look at the OSD log gives this:
A deeper look using ceph CLI tools gives:
and
and
There are two OSD pointing to the same disc (/dev/sdc), I guess because of a naming problem on reboot.
How could I fix this?
I have a Proxmox+Ceph cluster, 5 nodes (8 OSD per nodes).
This morning, I updated (to last 8.4) and rebooted each node (one at a time, moving the VM beforehand, etc).
I ended up with two "out" and "outdated" OSD (both "out" and "outdated"), one on node 3 and one on node 5.
Tried to restart OSD-34 (one of the two "out" OSD, this one is on node 5) manually, didn't work.
A quick look at the OSD log gives this:
Code:
2026-06-07T11:29:18.667+0200 7830cac68940 -1 OSD id 35 != my id 34
A deeper look using ceph CLI tools gives:
Code:
root@pve05:/var/log/ceph# ceph osd metadata osd.34 | egrep "id|objectstore"
"id": 34,
"device_ids": "sdc=VK1920GFLKL_FJ73N65941030452Q",
"objectstore_numa_unknown_devices": "sdc",
"osd_objectstore": "bluestore",
Code:
root@pve05:/var/log/ceph# ceph osd metadata osd.35 | egrep "id|objectstore"
"id": 35,
"device_ids": "sdc=VK1920GFLKL_FJ73N65941030452E",
"objectstore_numa_unknown_devices": "sdc",
"osd_objectstore": "bluestore",
Code:
root@pve05:/var/log/ceph# ceph-volume raw list
{
"04a9d59d-2161-48fb-b8c6-f60420eeefa9": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sde2",
"osd_id": 36,
"osd_uuid": "04a9d59d-2161-48fb-b8c6-f60420eeefa9",
"type": "bluestore"
},
"1b957d5c-9ec8-4933-b543-8a46c9b7a972": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sdf2",
"osd_id": 38,
"osd_uuid": "1b957d5c-9ec8-4933-b543-8a46c9b7a972",
"type": "bluestore"
},
"6729a36c-e26b-4577-ad53-9b6fda7bad8e": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sdh2",
"osd_id": 39,
"osd_uuid": "6729a36c-e26b-4577-ad53-9b6fda7bad8e",
"type": "bluestore"
},
"a220a826-b18f-43c1-886b-999b229abfca": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sda2",
"osd_id": 32,
"osd_uuid": "a220a826-b18f-43c1-886b-999b229abfca",
"type": "bluestore"
},
"a47518fa-cae3-4be9-a21d-af4dedb04828": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sdc2",
"osd_id": 35,
"osd_uuid": "a47518fa-cae3-4be9-a21d-af4dedb04828",
"type": "bluestore"
},
"b99ce6b7-7d81-47d4-b953-86cc736eba1d": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sdb2",
"osd_id": 33,
"osd_uuid": "b99ce6b7-7d81-47d4-b953-86cc736eba1d",
"type": "bluestore"
},
"be574925-28f2-4b8e-a0a3-783be99ec332": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sdd2",
"osd_id": 34,
"osd_uuid": "be574925-28f2-4b8e-a0a3-783be99ec332",
"type": "bluestore"
},
"ff7e748f-0d18-4776-adf7-d6a83d323242": {
"ceph_fsid": "d72132ab-739f-4782-be07-7d5980f8dabe",
"device": "/dev/sdg2",
"osd_id": 37,
"osd_uuid": "ff7e748f-0d18-4776-adf7-d6a83d323242",
"type": "bluestore"
}
}
There are two OSD pointing to the same disc (/dev/sdc), I guess because of a naming problem on reboot.
How could I fix this?
Last edited: