ceph.conf - Checking the configuration

[osalj]

Hi,

I have a Ceph cluster on three Dell R330 servers.

Public network is 2x 1Gbps in bond (LACP)
Ceph network is 10Gbps SFP+
Network for vms/lxc/internet is 10Gbps SFP+

I configured Ceph via Proxmox GUI, my ceph.conf file below:

[global]
auth_client_required = cephx
auth_cluster_required = cephx
auth_service_required = cephx
cluster_network = 10.41.41.41/24
fsid = 096444ae-b614-4430-8e12-01bbe4239a80
mon_allow_pool_delete = true
mon_host = 192.168.43.42 192.168.43.43 192.168.43.41
ms_bind_ipv4 = true
ms_bind_ipv6 = false
osd_pool_default_min_size = 2
osd_pool_default_size = 3
public_network = 192.168.43.41/24

[client]
keyring = /etc/pve/priv/$cluster.$name.keyring

[client.crash]
keyring = /etc/pve/ceph/$cluster.$name.keyring

[mon.pve01]
public_addr = 192.168.43.41

[mon.pve02]
public_addr = 192.168.43.42

[mon.pve03]
public_addr = 192.168.43.43

I wonder if cluster_network and public_network are set correctly?
I think it should be 10.41.41.0/24 for cluster_network and 192.168.43.0/24 for public_network.

Is the rest of my ceph.conf configuration file correct?

The ceph.conf file looks the same on every node.

I am also attaching a file with the network interface configuration:
[CODE}
auto lo
iface lo inet loopback

iface enp2s0f0 inet manual
#Interface for vmbr0

auto eno1
iface eno1 inet manual
#Interface for bond0

auto eno2
iface eno2 inet manual
#Interface for bond0

auto enp2s0f1
iface enp2s0f1 inet static
address 10.41.41.41/24
#Interface for CEPH

auto bond0
iface bond0 inet static
address 192.168.43.41/24
bond-slaves eno1 eno2
bond-miimon 100
bond-mode 802.3ad
#bond0 for PUBLIC network

auto vmbr0
iface vmbr0 inet static
address 10.40.1.41/24
gateway 10.40.1.1
bridge-ports enp2s0f0
bridge-stp off
bridge-fd 0
#Main bridge
[/CODE]

TIA!

 

[groque]

Your config is actually correct. cluster_network and public_network in ceph.conf must point to the node’s IP, not the subnet. Ceph binds to those IPs.

Here's an example from a working setup using with 10.10.2.0/24 as the Ceph public network and 10.10.3.0/24 as the Ceph cluster network:

[global]
    auth_client_required = cephx
    auth_cluster_required = cephx
    auth_service_required = cephx
    cluster_network = 10.10.3.11/24
    fsid = b5cc87df-afcb-4b95-8644-0a8855d5a16d
    mon_allow_pool_delete = true
    mon_host = 10.10.2.11 10.10.2.12 10.10.2.13 10.10.2.14 10.10.2.15
    ms_bind_ipv4 = true
    ms_bind_ipv6 = false
    osd_pool_default_min_size = 2
    osd_pool_default_size = 3
    public_network = 10.10.2.11/24

[client]
    keyring = /etc/pve/priv/$cluster.$name.keyring

[client.crash]
    keyring = /etc/pve/ceph/$cluster.$name.keyring

[mds]
    keyring = /var/lib/ceph/mds/ceph-$id/keyring


[mds.sm01a]
    host = sm01a
    mds_standby_for_name = pve

[mds.sm01b]
    host = sm01b
    mds_standby_for_name = pve

[mds.sm01c]
    host = sm01c
    mds_standby_for_name = pve

[mds.sm01d]
    host = sm01d
    mds_standby_for_name = pve

[mds.sm01e]
    host = sm01e
    mds_standby_for_name = pve


[mon.sm01a]
    public_addr = 10.10.2.11

[mon.sm01b]
    public_addr = 10.10.2.12

[mon.sm01c]
    public_addr = 10.10.2.13

[mon.sm01d]
    public_addr = 10.10.2.14

[mon.sm01e]
    public_addr = 10.10.2.15

 

[ucholak]

my config binds to networks "0/24". and config is same on all nodes (corosync), so there cant be same ip as in your case