Proxmox cluster iptables nat rules when using vpn

[leonidas_o]

Hello,
I'm experiencing a pretty strange connection issue when connected to the second vpn VM (wireguard).
I got two proxmox nodes, each do have a VM with wireguard, a VM with a dns server, a VM with traefik proxy, both nodes have the same etc/network/interfaces files.
Both wireguard VMs are clones of each other and configured the same, both has the following Interface setting: PostUp = iptables -A FORWARD -i tun0 -j ACCEPT; iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;

- Now when connected to wireguard1 VM on node1, everything is working fine, I'm redirecting everything through the wireguard tunnel, I can browse the internet, the port forwarding is working, means I can access any site which is behind traefik proxy etc.
- When connected to wireguard2 VM on node2, I can browse the internet, dns is working, but I can't enter the sites behind traefik proxy on node1. Just getting a Gateway Timeout. I can ping node1, but that's actually it. Seems like portforwarding is not working, or maybe the nat is not configured correctly.
- When not connected at all to a wireguard VM, so comming from the public internet, everything is working fine again. I can enter the public sites through traefik proxy.

So by either connected to a wireguard VM, which is on the same server as the traefik VM, or coming from the internet, everything works as expected, but as soon as connected to wireguard on node2, my services are not reachable anymore.
I removed the other port forwardings in here, because they behave the same like the ones for traefik. I think if I solve that, the other issues will be solved, too.
Somehow the connection coming from wireguard2 (node2) to node1 is not working at all, although I would expect that it behaves the same like coming from the internet. Both approaches should actually enter node1 over enp0s31f6, right? Maybe the traffic ist not routed properly somewhere in between.
Any idea if I'm missing any iptables rules?




etc/network/interfaces

auto lo
iface lo inet loopback
iface lo inet6 loopback

auto enp0s31f6
iface enp0s31f6 inet static
  address 88.XX.XX.XX
  netmask 255.255.255.192
  gateway 88.XX.XX.XX
  mtu 1500

iface enp0s31f6 inet6 static
  address 2aXX:XXX:XXX:XXX::2
  netmask 64
  gateway fXXX::1


auto vmbr0
iface vmbr0 inet static
        address 10.2.1.1/24
        bridge-ports none
        bridge-stp off
        bridge-fd 0

        post-up   echo 1 > /proc/sys/net/ipv4/ip_forward
        post-up   iptables -t nat -A POSTROUTING -s '10.2.1.0/24' -o enp0s31f6 -j MASQUERADE
        post-down iptables -t nat -D POSTROUTING -s '10.2.1.0/24' -o enp0s31f6 -j MASQUERADE
        post-up   iptables -t raw -I PREROUTING -i fwbr+ -j CT --zone 1
        post-down iptables -t raw -D PREROUTING  -i fwbr+ -j CT --zone 1
        # port forwarding
        # wireguard1
        post-up   iptables -t nat -A PREROUTING -i enp0s31f6 -p udp --dport XXXX -j DNAT --to 10.2.1.3:51820
        post-down iptables -t nat -D PREROUTING -i enp0s31f6 -p udp --dport XXXX -j DNAT --to 10.2.1.3:51820
        # ssh
        post-up   iptables -t nat -A PREROUTING -i enp0s31f6 -p tcp --dport XXXX -j DNAT --to 10.2.1.3:22
        post-down iptables -t nat -D PREROUTING -i enp0s31f6 -p tcp --dport XXXX -j DNAT --to 10.2.1.3:22

auto vmbr1
iface vmbr1 inet static
        address 10.2.2.1/24
        bridge-ports none
        bridge-stp off
        bridge-fd 0

        post-up   iptables -t nat -A POSTROUTING -s '10.2.2.0/24' -o enp0s31f6 -j MASQUERADE
        post-down iptables -t nat -D POSTROUTING -s '10.2.2.0/24' -o enp0s31f6 -j MASQUERADE
        # port forwarding
        ...
        # traefik
        post-up   iptables -t nat -A PREROUTING -i enp0s31f6 -p tcp --dport 80 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:8080
        post-down iptables -t nat -D PREROUTING -i enp0s31f6 -p tcp --dport 80 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:8080
        post-up   iptables -t nat -A PREROUTING -i vmbr0 -p tcp --dport 80 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:8080
        post-down iptables -t nat -D PREROUTING -i vmbr0 -p tcp --dport 80 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:8080
        post-up   iptables -t nat -A PREROUTING -i enp0s31f6 -p tcp --dport 443 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:4430
        post-down iptables -t nat -D PREROUTING -i enp0s31f6 -p tcp --dport 443 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:4430
        post-up   iptables -t nat -A PREROUTING -i vmbr0 -p tcp --dport 443 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:4430
        post-down iptables -t nat -D PREROUTING -i vmbr0 -p tcp --dport 443 -d mail.MYDOMAIN.com -j DNAT --to 10.2.2.2:4430
        post-up   iptables -t nat -A PREROUTING -i vmbr1 -p tcp --dport 443 -d MYDOMAIN.com -j DNAT --to 10.2.2.2:4430
        post-down iptables -t nat -D PREROUTING -i vmbr1 -p tcp --dport 443 -d MYDOMAIN.com -j DNAT --to 10.2.2.2:4430
        ...

auto vmbr2
iface vmbr2 inet static
        address 10.2.3.1/24
        bridge-ports none
        bridge-stp off
        bridge-fd 0

        post-up   echo 1 > /proc/sys/net/ipv4/ip_forward
        post-up   iptables -t nat -A POSTROUTING -s '10.2.3.0/24' -o enp0s31f6 -j MASQUERADE
        post-down iptables -t nat -D POSTROUTING -s '10.2.3.0/24' -o enp0s31f6 -j MASQUERADE



source /etc/network/interfaces.d/*