SDN slow reload with multiple VNETs

[the_upside_down]

I've set up a VXLAN zone in the SDN, and am programatically creating multiple VNETs in that zone. However this has quickly become a bottleneck as reloads are taking upwards of ten minutes to complete.

On any node (of the 9 in the cluster) `ifreload -a` completes in under 7 seconds, but each node takes 30 - 90 seconds to apply, which stack up as the SDN apply API call runs one node at a time.

I can't see any obvious reason why this is taking so long or how to cut it down, short of reducing the number of VNETs or redesigning the network architecture.

 

[tinduong1337]

try set `addon_scripts_support=0` in `/etc/network/ifupdown2/ifupdown2.conf` (it's 1 by default)

 

[the_upside_down]

I've updated that line in all the nodes, no luck unfortunately, still takes a long time to apply on each node.

 

[shanreich]

How does your VNet configuration, as well as your network configuration look like?

cat /etc/network/interfaces
cat /etc/network/interfaces.d/sdn
cat /etc/pve/sdn/zones.cfg
cat /etc/pve/sdn/vnets.cfg

 

[the_upside_down]

Note - /etc/network/interfaces.d/sdn is a snippet, there are a few hundred virtual NICs in there.

# network interface settings; autogenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!

auto lo
iface lo inet loopback

auto nic2
iface nic2 inet manual

iface nic4 inet manual

iface nic0 inet manual

auto nic1
iface nic1 inet manual

auto nic5
iface nic5 inet manual

iface nic6 inet manual
        mtu 9000

iface nic7 inet manual
        mtu 9000

iface nic3 inet manual

auto vmbr0
iface vmbr0 inet static
        address 10.150.0.8/24
        gateway 10.150.0.254
        bridge-ports nic0
        bridge-stp off
        bridge-fd 0
        bridge-vlan-aware yes
        bridge-vids 2-4094

auto vmbr2
iface vmbr2 inet static
        address 10.155.0.8/24
        bridge-ports nic6
        bridge-stp off
        bridge-fd 0
        mtu 9000

auto vmbr3
iface vmbr3 inet static
        address 10.156.0.8/24
        bridge-ports nic7
        bridge-stp off
        bridge-fd 0
        mtu 9000

auto vmbr1
iface vmbr1 inet static
        address 10.151.0.8/24
        bridge-ports nic3
        bridge-stp off
        bridge-fd 0
        bridge-vlan-aware yes
        bridge-vids 2-4094

source /etc/network/interfaces.d/*

auto vxlan_e10324
iface vxlan_e10324
        vxlan-id 10324
        vxlan_remoteip 10.151.0.4
        vxlan_remoteip 10.151.0.5
        vxlan_remoteip 10.151.0.6
        vxlan_remoteip 10.151.0.7
        vxlan_remoteip 10.151.0.9
        vxlan_remoteip 10.151.0.10
        vxlan_remoteip 10.151.0.11
        vxlan_remoteip 10.151.0.12
        mtu 1450

auto vxlan_e10325
iface vxlan_e10325
        vxlan-id 10325
        vxlan_remoteip 10.151.0.4
        vxlan_remoteip 10.151.0.5
        vxlan_remoteip 10.151.0.6
        vxlan_remoteip 10.151.0.7
        vxlan_remoteip 10.151.0.9
        vxlan_remoteip 10.151.0.10
        vxlan_remoteip 10.151.0.11
        vxlan_remoteip 10.151.0.12
        mtu 1450

auto vxlan_e10326
iface vxlan_e10326
        vxlan-id 10326
        vxlan_remoteip 10.151.0.4
        vxlan_remoteip 10.151.0.5
        vxlan_remoteip 10.151.0.6
        vxlan_remoteip 10.151.0.7
        vxlan_remoteip 10.151.0.9
        vxlan_remoteip 10.151.0.10
        vxlan_remoteip 10.151.0.11
        vxlan_remoteip 10.151.0.12
        mtu 1450

vxlan: mainlan
        ipam pve
        peers 10.151.0.4 10.151.0.5 10.151.0.6 10.151.0.7 10.151.0.8 10.151.0.9 10.151.0.10 10.151.0.11 10.151.0.12

vnet: e10046
        zone mainlan
        alias env instance 26 Salistron
        tag 10046

vnet: e10326
        zone mainlan
        alias env instance 28 LAN
        tag 10326

vnet: e10325
        zone mainlan
        alias env instance 28 WAN
        tag 10325

vnet: e10324
        zone mainlan
        alias env instance 28 PCAP
        tag 10324

vnet: e10323
        zone mainlan
        alias env instance 28 LAN
        tag 10323

vnet: e10322
        zone mainlan
        alias env instance 28 WAN
        tag 10322

vnet: e10321
        zone mainlan
        alias env instance 28 PCAP
        tag 10321