For node 2 (the working node) -
this has an Intel e1000e NIC and I had to disable NIC offloading (I implemented this fix https://gist.github.com/crypt0rr/60aaabd4a5c29a256b4f276122765237)
root@proxmox2:~# cat /etc/network/interfaces
# network interface settings; autogenerated
# Please do NOT modify this file directly, unless you know what
# you're doing.
#
# If you want to manage parts of the network configuration manually,
# please utilize the 'source' or 'source-directory' directives to do
# so.
# PVE will preserve these directives, but will NOT read its network
# configuration from sourced files, so do not attempt to move any of
# the PVE managed interfaces into external files!
auto lo
iface lo inet loopback
auto eno2
iface eno2 inet manual
# post-up ethtool -K eno2 gso off tso off rxvlan off txvlan off gro off tx off rx off sg off
auto vmbr0
=========================================
root@proxmox2:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host noprefixroute
valid_lft forever preferred_lft forever
2: eno2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master vmbr0 state UP group default qlen 1000
link/ether cc:48:3a:63:78:fa brd ff:ff:ff:ff:ff:ff
altname enp0s31f6
altname enxcc483a6378fa
3: wlo1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 94:e7:0b:7a:90:9a brd ff:ff:ff:ff:ff:ff
altname wlp0s20f3
altname wlx94e70b7a909a
4: vmbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether cc:48:3a:63:78:fa brd ff:ff:ff:ff:ff:ff
inet 192.168.1.132/24 scope global vmbr0
valid_lft forever preferred_lft forever
inet6 fe80::ce48:3aff:fe63:78fa/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
5: tap151i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master vmbr0 state UNKNOWN group default qlen 1000
link/ether 2a:d9:23:b5:a5:98 brd ff:ff:ff:ff:ff:ff
6: tap106i0: <BROADCAST,MULTICAST,PROMISC,UP,LOWER_UP> mtu 1500 qdisc fq_codel master fwbr106i0 state UNKNOWN group default qlen 1000
link/ether 8a:3e:fa:bb:20:31 brd ff:ff:ff:ff:ff:ff
7: fwbr106i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 4e:f5:50:c4:d5:be brd ff:ff:ff:ff:ff:ff
8: fwpr106p0@fwln106i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether 56:de:d8:07:b7:d8 brd ff:ff:ff:ff:ff:ff
9: fwln106i0@fwpr106p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr106i0 state UP group default qlen 1000
link/ether 4e:f5:50:c4:d5:be brd ff:ff:ff:ff:ff:ff
10: veth141i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr141i0 state UP group default qlen 1000
link/ether fe:54:cb:9b:e8:d6 brd ff:ff:ff:ff:ff:ff link-netnsid 0
11: fwbr141i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 6e:f7:31:af:fa:44 brd ff:ff:ff:ff:ff:ff
12: fwpr141p0@fwln141i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether d2:25:9c:f8:97:07 brd ff:ff:ff:ff:ff:ff
13: fwln141i0@fwpr141p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr141i0 state UP group default qlen 1000
link/ether 6e:f7:31:af:fa:44 brd ff:ff:ff:ff:ff:ff
14: veth142i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr142i0 state UP group default qlen 1000
link/ether fe:bb:74:5d:a3:0a brd ff:ff:ff:ff:ff:ff link-netnsid 1
15: fwbr142i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether ae:a3:83:1b:38:8a brd ff:ff:ff:ff:ff:ff
16: fwpr142p0@fwln142i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether f6:c4:a3:50:73:1a brd ff:ff:ff:ff:ff:ff
17: fwln142i0@fwpr142p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr142i0 state UP group default qlen 1000
link/ether ae:a3:83:1b:38:8a brd ff:ff:ff:ff:ff:ff
18: veth107i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr107i0 state UP group default qlen 1000
link/ether fe:4b:d7:e8:7b:fb brd ff:ff:ff:ff:ff:ff link-netnsid 2
19: fwbr107i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 02:e2:d2:de:0d:58 brd ff:ff:ff:ff:ff:ff
20: fwpr107p0@fwln107i0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether 6e:00:7a:93:20:6d brd ff:ff:ff:ff:ff:ff
21: fwln107i0@fwpr107p0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master fwbr107i0 state UP group default qlen 1000
link/ether 02:e2:d2:de:0d:58 brd ff:ff:ff:ff:ff:ff
22: veth103i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:e0:eb:04:45:f1 brd ff:ff:ff:ff:ff:ff link-netnsid 3
23: veth109i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:63:1a:9d:f1:2f brd ff:ff:ff:ff:ff:ff link-netnsid 4
24: veth100i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:e8:59:26:3d:d1 brd ff:ff:ff:ff:ff:ff link-netnsid 5
25: veth101i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:2f:1b:61:e5:8b brd ff:ff:ff:ff:ff:ff link-netnsid 6
26: veth162i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:71:9c:06:1a:11 brd ff:ff:ff:ff:ff:ff link-netnsid 7
64: veth111i0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master vmbr0 state UP group default qlen 1000
link/ether fe:54:fc:af:40:8d brd ff:ff:ff:ff:ff:ff link-netnsid 8
============================================
root@proxmox2:~# ip r
default via 192.168.1.1 dev vmbr0 proto kernel onlink
192.168.1.0/24 dev vmbr0 proto kernel scope link src 192.168.1.132
===========================================
root@proxmox2:~# lspci -nnk
00:00.0 Host bridge [0600]: Intel Corporation Comet Lake-U v1 4c Host Bridge/DRAM Controller [8086:9b61] (rev 0c)
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: skl_uncore
00:02.0 VGA compatible controller [0300]: Intel Corporation CometLake-U GT2 [UHD Graphics] [8086:9b41] (rev 02)
DeviceName: Onboard - Video
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: i915
Kernel modules: i915
00:04.0 Signal processing controller [1180]: Intel Corporation Xeon E3-1200 v5/E3-1500 v5/6th Gen Core Processor Thermal Subsystem [8086:1903] (rev 0c)
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: proc_thermal
Kernel modules: processor_thermal_device_pci_legacy
00:08.0 System peripheral [0880]: Intel Corporation Xeon E3-1200 v5/v6 / E3-1500 v5 / 6th/7th/8th Gen Core Processor Gaussian Mixture Model [8086:1911]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
00:12.0 Signal processing controller [1180]: Intel Corporation Comet Lake Thermal Subsytem [8086:02f9]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: intel_pch_thermal
Kernel modules: intel_pch_thermal
00:14.0 USB controller [0c03]: Intel Corporation Comet Lake PCH-LP USB 3.1 xHCI Host Controller [8086:02ed]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: xhci_hcd
Kernel modules: xhci_pci
00:14.2 RAM memory [0500]: Intel Corporation Comet Lake PCH-LP Shared SRAM [8086:02ef]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
00:14.3 Network controller [0280]: Intel Corporation Comet Lake PCH-LP CNVi WiFi [8086:02f0]
DeviceName: Onboard - Ethernet
Subsystem: Intel Corporation Dual Band Wi-Fi 6(802.11ax) AX201 160MHz 2x2 [Harrison Peak] [8086:4070]
Kernel driver in use: iwlwifi
Kernel modules: iwlwifi
00:15.0 Serial bus controller [0c80]: Intel Corporation Serial IO I2C Host Controller [8086:02e8]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: intel-lpss
Kernel modules: intel_lpss_pci
00:15.1 Serial bus controller [0c80]: Intel Corporation Comet Lake Serial IO I2C Host Controller [8086:02e9]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: intel-lpss
Kernel modules: intel_lpss_pci
00:16.0 Communication controller [0780]: Intel Corporation Comet Lake Management Engine Interface [8086:02e0]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: mei_me
Kernel modules: mei_me
00:16.3 Serial controller [0700]: Intel Corporation Comet Lake AMT SOL Redirection [8086:02e3]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: serial
Kernel modules: 8250_pci
00:17.0 SATA controller [0106]: Intel Corporation Comet Lake SATA AHCI Controller [8086:02d3]
DeviceName: Onboard - SATA
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: ahci
Kernel modules: ahci
00:19.0 Serial bus controller [0c80]: Intel Corporation Comet Lake Serial IO I2C Host Controller [8086:02c5]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: intel-lpss
Kernel modules: intel_lpss_pci
00:1c.0 PCI bridge [0604]: Intel Corporation Comet Lake PCI Express Root Port #1 [8086:02b8] (rev f0)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
00:1c.4 PCI bridge [0604]: Intel Corporation Comet Lake PCI Express Root Port #5 [8086:02bc] (rev f0)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
00:1d.0 PCI bridge [0604]: Intel Corporation Comet Lake PCI Express Root Port #13 [8086:02b4] (rev f0)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
00:1f.0 ISA bridge [0601]: Intel Corporation Comet Lake PCH-LP LPC Premium Controller/eSPI Controller [8086:0284]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
00:1f.3 Audio device [0403]: Intel Corporation Comet Lake PCH-LP cAVS [8086:02c8]
DeviceName: Onboard - Sound
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: snd_hda_intel
Kernel modules: snd_soc_avs, snd_sof_pci_intel_cnl, snd_hda_intel
00:1f.4 SMBus [0c05]: Intel Corporation Comet Lake PCH-LP SMBus Host Controller [8086:02a3]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: i801_smbus
Kernel modules: i2c_i801
00:1f.5 Serial bus controller [0c80]: Intel Corporation Comet Lake SPI (flash) Controller [8086:02a4]
DeviceName: Onboard - Other
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: intel-spi
Kernel modules: spi_intel_pci
00:1f.6 Ethernet controller [0200]: Intel Corporation Ethernet Connection (10) I219-LM [8086:0d4e]
DeviceName: Onboard - Ethernet
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: e1000e
Kernel modules: e1000e
01:00.0 Unassigned class [ff00]: Realtek Semiconductor Co., Ltd. RTS525A PCI Express Card Reader [10ec:525a] (rev 01)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: rtsx_pci
Kernel modules: rtsx_pci
02:00.0 PCI bridge [0604]: Intel Corporation JHL7540 Thunderbolt 3 Bridge [Titan Ridge 2C 2018] [8086:15e7] (rev 06)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
03:00.0 PCI bridge [0604]: Intel Corporation JHL7540 Thunderbolt 3 Bridge [Titan Ridge 2C 2018] [8086:15e7] (rev 06)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
03:01.0 PCI bridge [0604]: Intel Corporation JHL7540 Thunderbolt 3 Bridge [Titan Ridge 2C 2018] [8086:15e7] (rev 06)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
03:02.0 PCI bridge [0604]: Intel Corporation JHL7540 Thunderbolt 3 Bridge [Titan Ridge 2C 2018] [8086:15e7] (rev 06)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: pcieport
Kernel modules: shpchp
04:00.0 System peripheral [0880]: Intel Corporation JHL7540 Thunderbolt 3 NHI [Titan Ridge 2C 2018] [8086:15e8] (rev 06)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: thunderbolt
Kernel modules: thunderbolt
3a:00.0 USB controller [0c03]: Intel Corporation JHL7540 Thunderbolt 3 USB Controller [Titan Ridge 2C 2018] [8086:15e9] (rev 06)
Subsystem: Dell Device [1028:09a0]
Kernel driver in use: xhci_hcd
Kernel modules: xhci_pci
3b:00.0 Non-Volatile memory controller [0108]: Kingston Technology Company, Inc. NV2 NVMe SSD [E21T] (DRAM-less) [2646:5019]
Subsystem: Kingston Technology Company, Inc. NV2 NVMe SSD [E21T] (DRAM-less) [2646:5019]
Kernel driver in use: nvme
Kernel modules: nvme