Ceph configuration override

[brosky]

Hi,

I have a ceph cluster configured via proxmox and added object storage via radosgw package.

My cluster is unstable for the moment and I have random reboots - something that I'm investigating now.

My issue is that after some nodes are rebooted, the /etc/ceph/ceph.conf file is overwrited with a basic setup and I lose my radosgw client setup.

Example:

Basic config:
root@pve299:/var/lib/ceph/radosgw# cat /etc/pve/ceph.conf

[global]
     auth_client_required = cephx
     auth_cluster_required = cephx
     auth_service_required = cephx
     cluster_network = 10.0.40.0/24
     fsid = ca8ecd14-1cfa-48e4-9952-59c4cab1ecc3
     mon_allow_pool_delete = true
     mon_host = 10.0.40.97 10.0.40.98 10.0.40.36
     ms_bind_ipv4 = true
     ms_bind_ipv6 = false
     osd_pool_default_min_size = 2
     osd_pool_default_size = 3
     public_network = 10.0.40.0/24

[client]
     keyring = /etc/pve/priv/$cluster.$name.keyring

[mds]
     keyring = /var/lib/ceph/mds/ceph-$id/keyring

[mds.pve236]
     host = pve236
     mds_standby_for_name = pve

[mds.pve237]
     host = pve237
     mds standby for name = pve

[mds.pve298]
     host = pve298
     mds_standby_for_name = pve

[mon.pve236]
     public_addr = 10.0.40.36

[mon.pve297]
     public_addr = 10.0.40.97

[mon.pve298]
     public_addr = 10.0.40.98

Missing config:

[client.radosgw.pve236]
        host = pve236
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.36
[client.radosgw.pve237]
        host = pve237
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.37
[client.radosgw.pve238]
        host = pve238
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.38
[client.radosgw.pve293]
        host = pve293
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.93
[client.radosgw.pve295]
        host = pve295
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.95
[client.radosgw.pve297]
        host = pve297
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.97
[client.radosgw.pve298]
        host = pve298
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.98
[client.radosgw.pve299]
        host = pve299
        keyring = /etc/pve/priv/ceph.client.radosgw.keyring
        log file = /var/log/ceph/client.radosgw.$host.log
        rgw_dns_name = 10.230.31.99

How can I make the ceph.conf persistent ?

 

[gurubert]

All recent Ceph versions use just a basic ceph.conf which only contains the Ceph fsid and the MON IPs.
Everything else belongs into the configuration database which can be edited with "ceph config".