OVS RSTP 1Gbps+10Gbps interconnect mesh

[mnovi]

Hi!

I am building a 3 node testing cluster where each server has 2x 1Gbps ports and 2x 10Gbps ports. 1Gbps ports are connected to different switches within same network (simulating a real environment) and 10Gbps ports are connected directly between servers. I read the following guides and my hardware is connected almost the same as described in the first link (except that I have some cheap/dumb switches instead of Juniper ones):

• https://pve.proxmox.com/wiki/Open_v...RSTP.29_-_1Gbps_uplink.2C_10Gbps_interconnect
• https://pve.proxmox.com/wiki/Full_Mesh_Network_for_Ceph_Server#RSTP_Loop_Setup

What I want to achieve is connection to two networks:

• existing network on 10.10.10.0/24, which is managed by the router to which the mentioned two switches are connected
• virtual network on 10.0.0.0/24, where all three nodes would have IPs 10.0.0.1, 10.0.0.2 and 10.0.0.3; and there would also be a vlan50 for communication on the same 10.0.0.0/24 network between VMs - so this would only exist/be created on the nodes and not in the outer network

I have successfully got over the first point and I can connect to the 10.10.10.0/24 from all nodes, but I somehow cannot get through the second one. Here is my current config (/etc/network/interfaces):

auto lo
iface lo inet loopback

# 1gbps
auto enp5s0f0
iface enp5s0f0 inet static
        ovs_type OVSPort
        ovs_bridge vmbr0
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=20000 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true
        ovs_mtu 9000

# 1gbps (currently not configured due to the problems with second network)
iface enp5s0f1 inet manual

# 10gbps (connected to other node 1)
auto enp4s0f0
iface enp4s0f0 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr0
        ovs_mtu 9000
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=2000 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true

# 10gbps (connected to other node 2)
auto enp4s0f1
iface enp4s0f1 inet manual
        ovs_type OVSPort
        ovs_bridge vmbr0
        ovs_mtu 9000
        ovs_options tag=1 vlan_mode=native-untagged other_config:rstp-enable=true other_config:rstp-path-cost=2100 other_config:rstp-port-admin-edge=false other_config:rstp-port-auto-edge=false other_config:rstp-port-mcheck=true

auto vmbr0
iface vmbr0 inet manual
        ovs_type OVSBridge
        ovs_ports enp5s0f0 enp4s0f0 enp4s0f1 vlan1 vlan50
        ovs_mtu 9000
        up ovs-vsctl set Bridge ${IFACE} rstp_enable=true other_config:rstp-priority=32768 other_config:rstp-forward-delay=4 other_config:rstp-max-age=6
        post-up sleep 10

auto vlan1
iface vlan1 inet static
        address 10.10.10.15/24 # on other nodes there is either 10.10.10.13 or 10.10.10.14
        gateway 10.10.10.1
        ovs_type OVSIntPort
        ovs_bridge vmbr0
        ovs_options tag=1
        ovs_mtu 1500

auto vlan50
iface vlan50 inet static
        address 10.0.0.3/24 # on other nodes there is either 10.0.0.1 or 10.0.0.2
        ovs_type OVSIntPort
        ovs_bridge vmbr0
        ovs_options tag=50
        ovs_mtu 1500

Traffic over vlan1 is working as expected, meanwhile everything I try to do over vlan50 fails. If I try to ping other nodes on one of the servers I get Destination Host Unreachable error.

I am wondering what is missing from this config and what would be the right way to achieve connection directly between nodes on vlan50 (and later between VMs on that VLAN). Any help would be greatly appreciated. Thank you in advance!