vmbr0 suddenly stopped working on 2 nodes out of 3 Urgent request for help

Gerhard W. Recher · Nov 22, 2020

We have a productive cluster
suddenly networking on vmbr0 stopped on 2 member nodes node 1 is ok.

how to get this back to work ? any hints are welcome !

Code:

auto lo
iface lo inet loopback

iface enp69s0f0 inet manual
        mtu 9000

iface enp204s0f0 inet manual
        mtu 9000

iface enp204s0f1 inet manual
        mtu 9000

auto eth2
iface eth2 inet manual
        mtu 9000

auto eth3
iface eth3 inet manual
        mtu 9000

auto vmbr0
iface vmbr0 inet manual
        address 10.100.200.141/24
        gateway 10.100.200.1
        bridge-ports eth2
        bridge-stp off
        bridge-fd 0
        bridge-vlan-aware yes
        bridge-vids 2-4094
        mtu 9000
#public

auto vmbr1
iface vmbr1 inet manual
        bridge-ports enp204s0f0
        bridge-stp off
        bridge-fd 0
        bridge-vlan-aware yes
        bridge-vids 2-4094
        mtu 9000
#corosync1

auto vmbr2
iface vmbr2 inet manual
        bridge-ports enp204s0f1
        bridge-stp off
        bridge-fd 0
        bridge-vlan-aware yes
        bridge-vids 2-4094
        mtu 9000
#corosync2

auto vmbr2.111
iface vmbr2.111 inet static
        address 10.111.200.141/24
        mtu 9000
        vlan-id 111
#corosync2

auto vmbr1.110
iface vmbr1.110 inet static
        address 10.110.200.141/24
        mtu 9000
        vlan-id 110
#corosync1

auto vmbr3
iface vmbr3 inet manual
        bridge-ports eth3
        bridge-stp off
        bridge-fd 0
        bridge-vlan-aware yes
        bridge-vids 2-4094
        mtu 9000

auto vmbr3.112
iface vmbr3.112 inet static
        address 10.112.200.141/24
        mtu 9000
        vlan-id 112
#cluster

Code:

 on pve01
 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.100.200.1    0.0.0.0         UG        0 0          0 vmbr0
10.100.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr0
10.110.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr1.110
10.111.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr2.111
10.112.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr3.112

 ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: enp204s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr1 state UP group default qlen 1000
    link/ether 3c:ec:ef:47:75:a2 brd ff:ff:ff:ff:ff:ff
3: enp204s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr2 state UP group default qlen 1000
    link/ether 3c:ec:ef:47:75:a3 brd ff:ff:ff:ff:ff:ff
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr0 state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7e:a0 brd ff:ff:ff:ff:ff:ff
5: eth3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr3 state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7e:a1 brd ff:ff:ff:ff:ff:ff
7: vmbr2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:47:75:a3 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3eec:efff:fe47:75a3/64 scope link
       valid_lft forever preferred_lft forever
8: vmbr2.111@vmbr2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:47:75:a3 brd ff:ff:ff:ff:ff:ff
    inet 10.111.200.141/24 scope global vmbr2.111
       valid_lft forever preferred_lft forever
    inet6 fe80::3eec:efff:fe47:75a3/64 scope link
       valid_lft forever preferred_lft forever
9: vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:47:75:a2 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3eec:efff:fe47:75a2/64 scope link
       valid_lft forever preferred_lft forever
10: vmbr1.110@vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:47:75:a2 brd ff:ff:ff:ff:ff:ff
    inet 10.110.200.141/24 scope global vmbr1.110
       valid_lft forever preferred_lft forever
    inet6 fe80::3eec:efff:fe47:75a2/64 scope link
       valid_lft forever preferred_lft forever
11: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7e:a0 brd ff:ff:ff:ff:ff:ff
    inet 10.100.200.141/24 scope global vmbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::526b:4bff:fef3:7ea0/64 scope link
       valid_lft forever preferred_lft forever
12: vmbr3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7e:a1 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::526b:4bff:fef3:7ea1/64 scope link
       valid_lft forever preferred_lft forever
13: vmbr3.112@vmbr3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7e:a1 brd ff:ff:ff:ff:ff:ff
    inet 10.112.200.141/24 scope global vmbr3.112
       valid_lft forever preferred_lft forever
    inet6 fe80::526b:4bff:fef3:7ea1/64 scope link
       valid_lft forever preferred_lft forever

 arp -an
? (10.100.200.1) at 00:22:55:3e:d6:04 [ether] on vmbr0
? (10.100.200.91) at 3c:07:71:78:69:40 [ether] on vmbr0
? (10.100.200.89) at 0c:42:a1:a4:92:f4 [ether] on vmbr0
? (10.100.200.101) at 00:0c:29:06:55:64 [ether] on vmbr0
? (10.100.200.133) at 00:30:48:2e:9a:2d [ether] on vmbr0
? (10.100.200.222) at <incomplete> on vmbr0
? (10.100.200.142) at <incomplete> on vmbr0
? (10.100.200.143) at 50:6b:4b:f3:7d:d0 [ether] on vmbr0
? (10.110.200.142) at 3c:ec:ef:46:39:46 [ether] on vmbr1.110
? (10.110.200.143) at 3c:ec:ef:47:77:c2 [ether] on vmbr1.110
? (10.111.200.142) at 3c:ec:ef:46:39:47 [ether] on vmbr2.111
? (10.111.200.143) at 3c:ec:ef:47:77:c3 [ether] on vmbr2.111
? (10.112.200.143) at 50:6b:4b:f3:7d:d1 [ether] on vmbr3.112
? (10.112.200.142) at 50:6b:4b:f3:7d:f1 [ether] on vmbr3.112


 ping 10.100.200.1
PING 10.100.200.1 (10.100.200.1) 56(84) bytes of data.
64 bytes from 10.100.200.1: icmp_seq=1 ttl=255 time=0.481 ms
64 bytes from 10.100.200.1: icmp_seq=2 ttl=255 time=0.506 ms

ping to pve02/pve03 not possible

Code:

 on pve02
 netstat -rn
Kernel IP routing table
Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
0.0.0.0         10.100.200.1    0.0.0.0         UG        0 0          0 vmbr0
10.100.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr0
10.110.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr1.110
10.111.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr2.111
10.112.200.0    0.0.0.0         255.255.255.0   U         0 0          0 vmbr3.112

 arp -an
? (10.112.200.141) at 50:6b:4b:f3:7e:a1 [ether] on vmbr3.112
? (10.100.200.143) at <incomplete> on vmbr0
? (10.100.200.222) at <incomplete> on vmbr0
? (10.111.200.143) at 3c:ec:ef:47:77:c3 [ether] on vmbr2.111
? (10.100.200.141) at <incomplete> on vmbr0
? (10.110.200.141) at 3c:ec:ef:47:75:a2 [ether] on vmbr1.110
? (10.111.200.141) at 3c:ec:ef:47:75:a3 [ether] on vmbr2.111
? (10.110.200.143) at 3c:ec:ef:47:77:c2 [ether] on vmbr1.110

root@pve02:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: enp204s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr1 state UP group default qlen 1000
    link/ether 3c:ec:ef:46:39:46 brd ff:ff:ff:ff:ff:ff
3: enp204s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr2 state UP group default qlen 1000
    link/ether 3c:ec:ef:46:39:47 brd ff:ff:ff:ff:ff:ff
4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr0 state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7d:f0 brd ff:ff:ff:ff:ff:ff
5: eth3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc mq master vmbr3 state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7d:f1 brd ff:ff:ff:ff:ff:ff
7: vmbr2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:46:39:47 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3eec:efff:fe46:3947/64 scope link
       valid_lft forever preferred_lft forever
8: vmbr2.111@vmbr2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:46:39:47 brd ff:ff:ff:ff:ff:ff
    inet 10.111.200.142/24 scope global vmbr2.111
       valid_lft forever preferred_lft forever
    inet6 fe80::3eec:efff:fe46:3947/64 scope link
       valid_lft forever preferred_lft forever
9: vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:46:39:46 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3eec:efff:fe46:3946/64 scope link
       valid_lft forever preferred_lft forever
10: vmbr1.110@vmbr1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 3c:ec:ef:46:39:46 brd ff:ff:ff:ff:ff:ff
    inet 10.110.200.142/24 scope global vmbr1.110
       valid_lft forever preferred_lft forever
    inet6 fe80::3eec:efff:fe46:3946/64 scope link
       valid_lft forever preferred_lft forever
11: vmbr3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7d:f1 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::526b:4bff:fef3:7df1/64 scope link
       valid_lft forever preferred_lft forever
12: vmbr3.112@vmbr3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7d:f1 brd ff:ff:ff:ff:ff:ff
    inet 10.112.200.142/24 scope global vmbr3.112
       valid_lft forever preferred_lft forever
    inet6 fe80::526b:4bff:fef3:7df1/64 scope link
       valid_lft forever preferred_lft forever
15: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9000 qdisc noqueue state UP group default qlen 1000
    link/ether 50:6b:4b:f3:7d:f0 brd ff:ff:ff:ff:ff:ff
    inet 10.100.200.142/24 scope global vmbr0
       valid_lft forever preferred_lft forever
    inet6 fe80::526b:4bff:fef3:7df0/64 scope link
       valid_lft forever preferred_lft forever

Code:

 Version on all three nodes are equal!
 pveversion -v
proxmox-ve: 6.2-2 (running kernel: 5.4.73-1-pve)
pve-manager: 6.2-15 (running version: 6.2-15/48bd51b6)
pve-kernel-5.4: 6.3-1
pve-kernel-helper: 6.3-1
pve-kernel-5.4.73-1-pve: 5.4.73-1
pve-kernel-5.4.65-1-pve: 5.4.65-1
pve-kernel-5.4.34-1-pve: 5.4.34-2
ceph: 14.2.11-pve1
ceph-fuse: 14.2.11-pve1
corosync: 3.0.4-pve1
criu: 3.11-3
glusterfs-client: 5.5-3
ifupdown: residual config
ifupdown2: 3.0.0-1+pve3
ksm-control-daemon: 1.3-1
libjs-extjs: 6.0.1-10
libknet1: 1.16-pve1
libproxmox-acme-perl: 1.0.5
libpve-access-control: 6.1-3
libpve-apiclient-perl: 3.0-3
libpve-common-perl: 6.2-4
libpve-guest-common-perl: 3.1-3
libpve-http-server-perl: 3.0-6
libpve-storage-perl: 6.2-10
libqb0: 1.0.5-1
libspice-server1: 0.14.2-4~pve6+1
lvm2: 2.03.02-pve4
lxc-pve: 4.0.3-1
lxcfs: 4.0.3-pve3
novnc-pve: 1.1.0-1
proxmox-backup-client: 1.0.1-1
proxmox-mini-journalreader: 1.1-1
proxmox-widget-toolkit: 2.3-10
pve-cluster: 6.2-1
pve-container: 3.2-3
pve-docs: 6.2-6
pve-edk2-firmware: 2.20200531-1
pve-firewall: 4.1-3
pve-firmware: 3.1-3
pve-ha-manager: 3.1-1
pve-i18n: 2.2-2
pve-qemu-kvm: 5.1.0-6
pve-xtermjs: 4.7.0-2
qemu-server: 6.2-20
smartmontools: 7.1-pve2
spiceterm: 3.1-1
vncterm: 1.6-2
zfsutils-linux: 0.8.5-pve1

Search

Search

vmbr0 suddenly stopped working on 2 nodes out of 3 Urgent request for help

Gerhard W. Recher

Well-Known Member