OVS Bridge 3 nodes for ceph only

[Grille]

Hi all,

I try to interconnect 3 pve nodes for testing ceph using an ovs bridge like described in example 4.
The packages ifupdown2 and openvswitch-switch are installed on every node.

Now I'am a bit lost to ping between all 3 nodes. Maybe it's blocked by faulty rtsp configuration?
The stupid case, it works until a reboot of PM1 and PM3.
The PVE-Firewall is not in use on all 3 nodes.
All nodes are running 7.1-9.

IP Subnet 10.55.55.0/24
VLAN 55

|-------|               |-------|
| G G X-------------------X G G |
|     X |               | X     |
|------\|               |/------|
   PM1  \               /  PM3
 .55.60  \             B .55.62
          \           /
           \|---  --|/
            \  G G  /
            |X     X|
            |-------|
               PM5
             .55.64

Ping PM1 -> PM3 not possible
Ping PM1 -> PM5 ok
Ping PM3 -> PM1 not possible
Ping PM3 -> PM5 ok
Ping PM5-> PM1 ok
Ping PM5 -> PM3 ok

In case I remove the interconnection cable between PM1<->PM5 I am still able to ping from PM1 to PM5 (over PM3).
In case I remove the interconnection cable between PM3<->PM5 I am still able to ping from PM3 to PM5 (over PM1).

PM1:~# ovs-vsctl get Bridge vmbr3 rstp_status
{rstp_bridge_id="8.000.40a6b7848568", rstp_bridge_port_id="8002", rstp_designated_id="8.000.40a6b7847968", rstp_designated_port_id="8001", rstp_root_id="8.000.40a6b7847968", rstp_root_path_cost="2100"}

PM3:~# ovs-vsctl get Bridge vmbr3 rstp_status
{rstp_bridge_id="8.000.40a6b7847968", rstp_bridge_port_id="0000", rstp_designated_id="8.000.40a6b7847968", rstp_designated_port_id="0000", rstp_root_id="8.000.40a6b7847968", rstp_root_path_cost="0"}
    
PM5:~# ovs-vsctl get Bridge vmbr3 rstp_status
{rstp_bridge_id="8.000.40a6b7848570", rstp_bridge_port_id="8002", rstp_designated_id="8.000.40a6b7847968", rstp_designated_port_id="8002", rstp_root_id="8.000.40a6b7847968", rstp_root_path_cost="6000"}

# /etc/network/interfaces
# PM5

auto enp1s0f0
iface enp1s0f0 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr3
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=6000 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true

auto enp1s0f1
iface enp1s0f1 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr3
        ovs_mtu 8996
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=6100 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true

auto vlan55
iface vlan55 inet static
        address 10.55.55.64/24
        ovs_type OVSIntPort
        ovs_bridge vmbr3
        ovs_mtu 8996
        ovs_options tag=55

auto vmbr3
iface vmbr3 inet manual
        ovs_type OVSBridge
        ovs_ports enp1s0f0 enp1s0f1 vlan55
        ovs_mtu 8996
        up ovs-vsctl set Bridge ${IFACE} rstp_enable=true other_config:rstp-priority=32768 other_config:rstp-forward-delay=4 other_config:rstp-max-age=6
        post-up sleep 10

# /etc/network/interfaces
# PM3

auto enp130s0f0
iface enp130s0f0 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr3
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=4000 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true

auto enp130s0f1
iface enp130s0f1 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr3
        ovs_mtu 8996
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=4100 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true
     
auto vlan55
iface vlan55 inet static
        address 10.55.55.62/24
        ovs_type OVSIntPort
        ovs_bridge vmbr3
        ovs_mtu 8996
        ovs_options tag=55

auto vmbr3
iface vmbr3 inet manual
        ovs_type OVSBridge
        ovs_ports enp130s0f0 enp130s0f1 vlan55
        ovs_mtu 8996
        up ovs-vsctl set Bridge ${IFACE} rstp_enable=true other_config:rstp-priority=32768 other_config:rstp-forward-delay=4 other_config:rstp-max-age=6
        post-up sleep 10

# /etc/network/interfaces
# PM1

auto enp133s0f0
iface enp133s0f0 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr3
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=2000 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true

auto enp133s0f1
iface enp133s0f1 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr3
        ovs_mtu 8996
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=2100 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true
      
auto vlan55
iface vlan55 inet static
        address 10.55.55.60/24
        ovs_type OVSIntPort
        ovs_bridge vmbr3
        ovs_mtu 8996
        ovs_options tag=55

auto vmbr3
iface vmbr3 inet manual
        ovs_type OVSBridge
        ovs_ports enp133s0f0 enp133s0f1 vlan55
        ovs_mtu 8996
        up ovs-vsctl set Bridge ${IFACE} rstp_enable=true other_config:rstp-priority=32768 other_config:rstp-forward-delay=4 other_config:rstp-max-age=6
        post-up sleep 10