I had a 4th host in my cluster for a few days, but it turns out the box was apparently just not up to snuff and was causing reliability issues. I decided to remove it. I used the "destroy" button to remove the monitor, I removed the host from CEPH, and from the overall proxmox cluster. Everything seemed to be fine. Last night I got a new box that matched the specs of the existing other hosts and decided to add it in as the new fourth host in the cluster. "prox-4" Basic adding, and most of CEPH setup perfectly.
I went to add a Monitor for it, and ran into all sorts of problems. The monitor adds, but wont start or at least stay running. I did a lot of troubleshooting and found when I ran:
Any advice, or assistance would be welcome. Thanks!
I went to add a Monitor for it, and ran into all sorts of problems. The monitor adds, but wont start or at least stay running. I did a lot of troubleshooting and found when I ran:
ceph --admin-daemon /var/run/ceph/ceph-mon.prox-4.asok mon_status monitor mon.prox-4 seems to always get re-assigned the rank of 3 ( I assume by other members of the monitor pool?) so it won't run. I assume this is due to the former host being removed and being added as a new host with the same name. Is there a good/safe way to purge all of the ceph monitors of a stale "rank 3" ranking for an old host so I could add prox-4 back as a monitor if I need to? Any advice, or assistance would be welcome. Thanks!
JSON:
{
"name": "prox-4",
"rank": 3,
"state": "probing",
"election_epoch": 0,
"uptime": 123582,
"quorum": [],
"features": {
"required_con": "2449958197560098820",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging",
"quincy",
"reef",
"squid"
],
"quorum_con": "0",
"quorum_mon": []
},
"outside_quorum": [
"prox-4"
],
"extra_probe_peers": [],
"sync_provider": [],
"monmap": {
"epoch": 0,
"fsid": "94601a0c-451c-4e5a-b3f6-3eed8dc33c7c",
"modified": "2025-07-24T03:12:35.826293Z",
"created": "2025-02-15T20:41:38.200705Z",
"min_mon_release": 19,
"min_mon_release_name": "squid",
"election_strategy": 1,
"disallowed_leaders: ": "",
"stretch_mode": false,
"tiebreaker_mon": "",
"removed_ranks: ": "3",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging",
"quincy",
"reef",
"squid"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "prox-1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "192.168.10.41:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "192.168.10.41:6789",
"nonce": 0
}
]
},
"addr": "192.168.10.41:6789/0",
"public_addr": "192.168.10.41:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 1,
"name": "prox-2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "192.168.10.42:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "192.168.10.42:6789",
"nonce": 0
}
]
},
"addr": "192.168.10.42:6789/0",
"public_addr": "192.168.10.42:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 2,
"name": "prox-3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "192.168.10.43:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "192.168.10.43:6789",
"nonce": 0
}
]
},
"addr": "192.168.10.43:6789/0",
"public_addr": "192.168.10.43:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
},
{
"rank": 3,
"name": "prox-4",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "192.168.10.44:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "192.168.10.44:6789",
"nonce": 0
}
]
},
"addr": "192.168.10.44:6789/0",
"public_addr": "192.168.10.44:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3f03cffffffdffff",
"release": "squid",
"num": 1
}
]
},
"stretch_mode": false
}