Vlan interface can not ping Storage server IP

Aug 7, 2024
5
0
1
I have identical HOST servers that I have configured with a IP addresses on a vlan. They can ping each other. One of them can ping my secondary storage server. The other can not. When I ping from the HOST that can not, I get zero results from the ping. Interesting is that I can have a VM on that host and have that VM be able to reach the Storage server. Another interesting discovery is that I can ping the storage server for 10 seconds if I switch the active NIC on that HOST to another that is connected to my backup switch. Does my settings look right?


Code:
# network interface settings; autogenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!

auto lo
iface lo inet loopback

auto nic0
iface nic0 inet manual
#Management NIC-A

auto nic1
iface nic1 inet manual
#Management NIC-B

auto nic2
iface nic2 inet manual
#Production NIC-A

auto nic3
iface nic3 inet manual
    mtu 9000
#NFS NIC-A

auto nic4
iface nic4 inet manual
#Production NIC-B

auto nic5
iface nic5 inet manual
    mtu 9000
#NFS NIC-B

auto bond0
iface bond0 inet manual
    bond-slaves nic0 nic1
    bond-miimon 100
    bond-mode active-backup
    bond-primary nic0
#Management BOND

auto bond1
iface bond1 inet manual
    bond-slaves nic2 nic4
    bond-miimon 100
    bond-mode active-backup
    bond-primary nic2
#Production BOND

auto bond2
iface bond2 inet manual
    bond-slaves nic3 nic5
    bond-miimon 100
    bond-mode active-backup
    bond-primary nic3
    mtu 9000
#NFS BOND

auto vmbr0
iface vmbr0 inet static
    address 10.0.5.52/16
    gateway 10.0.0.1
    bridge-ports bond0
    bridge-stp off
    bridge-fd 0
#Management Bridge[IP]

auto production
iface production inet manual
    bridge-ports bond1
    bridge-stp off
    bridge-fd 0
    bridge-vlan-aware yes
    bridge-vids 2-4094
#Production Bridge

auto vm_network
iface vm_network inet manual
    bridge-ports production.5
    bridge-stp off
    bridge-fd 0
#VM Network Bridge

auto guest_net
iface guest_net inet manual
    bridge-ports production.3529
    bridge-stp off
    bridge-fd 0
#Guest Network

auto vlan200
iface vlan200 inet manual
    bridge-ports production.200
    bridge-stp off
    bridge-fd 0
#Production VLAN200

auto vlan3010
iface vlan3010 inet manual
    bridge-ports production.3010
    bridge-stp off
    bridge-fd 0
#Production VLAN3010

auto nfs
iface nfs inet manual
    bridge-ports bond2
    bridge-stp off
    bridge-fd 0
    bridge-vlan-aware yes
    bridge-vids 2-4094
    mtu 9000
#NFS Interface

auto nfs.999
iface nfs.999 inet static
    address 192.168.99.52/24
    mtu 1500
#NFS Interface Secondary [IP]

auto nfs.2208
iface nfs.2208 inet static
    address 172.22.208.112/24
    mtu 9000
#NFS Primary [IP]

source /etc/network/interfaces.d/*

When I do

Code:
ip neigh | grep 999
192.168.99.110 dev nfs.999 lladdr d2:39:ea:a8:a1:1e REACHABLE

But when i ping

Code:
ping  192.168.99.110
PING 192.168.99.110 (192.168.99.110) 56(84) bytes of data.

I get nothing. The host that I can ping from has results. Not sure how to troubleshoot this further.
 
Is this the configuration from the host that doesn't work? Could you post the /etc/network/interfaces from the other host as well?
Could you also post the current running state of the network configuration of both hosts? (ip a)
 
Thank you for taking a look at my settings!
This is the interface of the working server

Code:
# network interface settings; autogenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!

auto lo
iface lo inet loopback

auto nic0
iface nic0 inet manual
#Management NIC-A

auto nic1
iface nic1 inet manual
#Management NIC-B

auto nic2
iface nic2 inet manual
#Production NIC-A

auto nic3
iface nic3 inet manual
    mtu 9000
#NFS NIC-A

auto nic4
iface nic4 inet manual
#Production NIC-B

auto nic5
iface nic5 inet manual
    mtu 9000
#NFS NIC-B

auto bond0
iface bond0 inet manual
    bond-slaves nic0 nic1
    bond-miimon 100
    bond-mode active-backup
    bond-primary nic0
#Management BOND

auto bond1
iface bond1 inet manual
    bond-slaves nic2 nic4
    bond-miimon 100
    bond-mode active-backup
    bond-primary nic2

#Production BOND

auto bond2
iface bond2 inet manual
    bond-slaves nic3 nic5
    bond-miimon 100
    bond-mode active-backup
    bond-primary nic3
    mtu 9000
#NFS BOND

auto vmbr0
iface vmbr0 inet static
    address 10.0.5.51/16
    gateway 10.0.0.1
    bridge-ports bond0
    bridge-stp off
    bridge-fd 0
#Management Bridge[IP]

auto nfs
iface nfs inet manual
    bridge-ports bond2
    bridge-stp off
    bridge-fd 0
    bridge-vlan-aware yes
    bridge-vids 2-4094
    mtu 9000
#NFS BRIDGE



auto nfs.999
iface nfs.999 inet static
    address 192.168.99.51/24
    mtu 1500
#NFS Interface Secondary [IP]

auto nfs.2208
iface nfs.2208 inet static
    address 172.22.208.111/24
    mtu 9000
#NFS Primary [IP]



auto production
iface production inet manual
    bridge-ports bond1
    bridge-stp off
    bridge-fd 0
    bridge-vlan-aware yes
    bridge-vids 2-4094
#Production BRIDGE

auto vm_network
iface vm_network inet manual
    bridge-ports production.5
    bridge-stp off
    bridge-fd 0
#VM Network Bridge

auto guest_net
iface guest_net inet manual
    bridge-ports production.3529
    bridge-stp off
    bridge-fd 0
#Guest Network

auto vlan200
iface vlan200 inet manual
    bridge-ports production.200
    bridge-stp off
    bridge-fd 0
#Production VLAN200

auto vlan3010
iface vlan3010 inet manual
    bridge-ports production.3010
    bridge-stp off
    bridge-fd 0
#Production VLAN3010

source /etc/network/interfaces.d/*
 
And This is the `ip a` of the working server

Code:
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host noprefixroute
       valid_lft forever preferred_lft forever
2: nic0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
    link/ether 34:48:ed:ed:06:60 brd ff:ff:ff:ff:ff:ff
    altname enp193s0f0
    altname enx3448eded0660
3: nic1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
    link/ether 34:48:ed:ed:06:60 brd ff:ff:ff:ff:ff:ff permaddr 34:48:ed:ed:06:61
    altname enp193s0f1
    altname enx3448eded0661
4: nic2: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond1 state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
    altname enp2s0f0np0
    altname enx2cea7f470aed
5: nic3: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond2 state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ee brd ff:ff:ff:ff:ff:ff
    altname enp2s0f1np1
    altname enx2cea7f470aee
6: nic4: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond1 state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff permaddr f8:f2:1e:a5:71:30
    altname enp65s0f0np0
    altname enxf8f21ea57130
7: nic5: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond2 state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ee brd ff:ff:ff:ff:ff:ff permaddr f8:f2:1e:a5:71:31
    altname enp65s0f1np1
    altname enxf8f21ea57131
8: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
    link/ether 34:48:ed:ed:06:60 brd ff:ff:ff:ff:ff:ff
9: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 34:48:ed:ed:06:60 brd ff:ff:ff:ff:ff:ff
    inet 10.0.5.51/16 scope global vmbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::3648:edff:feed:660/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
10: bond2: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 9000 qdisc noqueue master nfs state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ee brd ff:ff:ff:ff:ff:ff
11: nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ee brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe47:aee/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
12: nfs.999@nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ee brd ff:ff:ff:ff:ff:ff
    inet 192.168.99.51/24 scope global nfs.999
       valid_lft forever preferred_lft forever
    inet6 fe80::2eea:7fff:fe47:aee/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
13: nfs.2208@nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ee brd ff:ff:ff:ff:ff:ff
    inet 172.22.208.111/24 scope global nfs.2208
       valid_lft forever preferred_lft forever
    inet6 fe80::2eea:7fff:fe47:aee/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
14: bond1: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue master production state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
15: production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe47:aed/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
16: production.5@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vm_network state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
17: vm_network: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe47:aed/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
18: production.3529@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master guest_net state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
19: guest_net: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe47:aed/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
20: production.200@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vlan200 state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
21: vlan200: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe47:aed/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
22: production.3010@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vlan3010 state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
23: vlan3010: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:47:0a:ed brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe47:aed/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
48: tap102i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether b2:54:3c:3b:3b:4e brd ff:ff:ff:ff:ff:ff
49: tap121i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether fe:9d:67:fa:4a:23 brd ff:ff:ff:ff:ff:ff
50: tap120i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether 6a:4a:37:4d:fc:e5 brd ff:ff:ff:ff:ff:ff
51: tap151i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether 76:46:66:a6:de:ef brd ff:ff:ff:ff:ff:ff
52: tap127i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether aa:ef:ce:88:36:12 brd ff:ff:ff:ff:ff:ff
53: tap154i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether 52:34:46:dc:8b:31 brd ff:ff:ff:ff:ff:ff
54: tap153i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether c2:cf:e2:e2:4f:9a brd ff:ff:ff:ff:ff:ff
55: tap150i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether 66:87:78:3a:f4:45 brd ff:ff:ff:ff:ff:ff


And the `ip a` of the not working server



Code:
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host noprefixroute
       valid_lft forever preferred_lft forever
2: nic0: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
    link/ether 34:48:ed:ec:ff:10 brd ff:ff:ff:ff:ff:ff
    altname enp193s0f0
    altname enx3448edecff10
3: nic1: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond0 state UP group default qlen 1000
    link/ether 34:48:ed:ec:ff:10 brd ff:ff:ff:ff:ff:ff permaddr 34:48:ed:ec:ff:11
    altname enp193s0f1
    altname enx3448edecff11
4: nic2: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond1 state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
    altname enp2s0f0np0
    altname enx2cea7f4ba8e6
5: nic3: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond2 state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e7 brd ff:ff:ff:ff:ff:ff
    altname enp2s0f1np1
    altname enx2cea7f4ba8e7
6: nic4: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 1500 qdisc mq master bond1 state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff permaddr f8:f2:1e:a5:8d:b0
    altname enp65s0f0np0
    altname enxf8f21ea58db0
7: nic5: <BROADCAST,MULTICAST,SLAVE,UP,LOWER_UP> mtu 9000 qdisc mq master bond2 state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e7 brd ff:ff:ff:ff:ff:ff permaddr f8:f2:1e:a5:8d:b1
    altname enp65s0f1np1
    altname enxf8f21ea58db1
8: bond0: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
    link/ether 34:48:ed:ec:ff:10 brd ff:ff:ff:ff:ff:ff
9: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 34:48:ed:ec:ff:10 brd ff:ff:ff:ff:ff:ff
    inet 10.0.5.52/16 scope global vmbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::3648:edff:feec:ff10/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
10: bond1: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 1500 qdisc noqueue master production state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
11: production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe4b:a8e6/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
12: production.5@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vm_network state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
13: vm_network: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe4b:a8e6/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
14: production.3529@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master guest_net state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
15: guest_net: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe4b:a8e6/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
16: production.200@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vlan200 state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
17: vlan200: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe4b:a8e6/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
18: production.3010@production: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vlan3010 state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
19: vlan3010: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e6 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe4b:a8e6/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
20: bond2: <BROADCAST,MULTICAST,MASTER,UP,LOWER_UP> mtu 9000 qdisc noqueue master nfs state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e7 brd ff:ff:ff:ff:ff:ff
21: nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e7 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::2eea:7fff:fe4b:a8e7/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
23: nfs.2208@nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e7 brd ff:ff:ff:ff:ff:ff
    inet 172.22.208.112/24 scope global nfs.2208
       valid_lft forever preferred_lft forever
    inet6 fe80::2eea:7fff:fe4b:a8e7/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
92: tap999i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master production state UNKNOWN group default qlen 1000
    link/ether 0e:66:5e:4a:d7:02 brd ff:ff:ff:ff:ff:ff
93: tap999i1: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 9000 qdisc fq_codel master fwbr999i1 state UNKNOWN group default qlen 1000
    link/ether ee:7d:07:bd:d5:d0 brd ff:ff:ff:ff:ff:ff
94: fwbr999i1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether c6:04:d4:2a:2c:53 brd ff:ff:ff:ff:ff:ff
95: fwpr999p1@fwln999i1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue master nfs state UP group default qlen 1000
    link/ether 62:15:84:37:d0:ee brd ff:ff:ff:ff:ff:ff
96: fwln999i1@fwpr999p1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue master fwbr999i1 state UP group default qlen 1000
    link/ether c6:04:d4:2a:2c:53 brd ff:ff:ff:ff:ff:ff
98: nfs.999@nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 2c:ea:7f:4b:a8:e7 brd ff:ff:ff:ff:ff:ff
    inet 192.168.99.52/24 scope global nfs.999
       valid_lft forever preferred_lft forever
    inet6 fe80::2eea:7fff:fe4b:a8e7/64 scope link proto kernel_ll
       valid_lft forever preferred_lft forever
 
Sorry for the late response, I took a look at the network configuration but couldn't really find anything off. Re-reading your initial post, the following part seems suspicious:

Another interesting discovery is that I can ping the storage server for 10 seconds if I switch the active NIC on that HOST to another that is connected to my backup switch. Does my settings look right?

That could indicate potentially duplicate IPs. Did you also check the port on the switch itself?
It sounds like the problem is somewhere in the network itself, rather than on the PVE hosts.
 
One thing that seems interesting to me is:
Bash:
nfs.2208@nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu [B]9000[/B] ...
nfs.999@nfs: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu [B]1500[/B] ...

Is there a reason these NFS VLANs have different MTUs? I'm not saying right or wrong, just this is an unusual configuration for me, so I'm curious.

The second, regarding
Another interesting discovery is that I can ping the storage server for 10 seconds if I switch the active NIC on that HOST to another that is connected to my backup switch.

How do you "switch the active NIC"? If you use something like:

Bash:
# echo nic5 | sudo tee /sys/class/net/bond2/bonding/primary

In a while, because
Code:
bond-primary nic3
and
Code:
primary_reselect=0
(which means "always" and this is the default value), the bond will switch back to nic3 (if miimon detects that the physical link of nic3 is UP).

So, at this moment that are the first things that pop up in my head. But without further investigation (reviewing the logs, checking the configurations of both switches and NFS servers, and seeing the output of tools like tcpdump), we are flying blind.

Best regards,
NT