Firewall is not working at cluster and node level

[m0n0pr1x]

Hi guys,
I am using Proxmox 8.0.4 and really enjoying it so far.

I tried to set up the firewall at the cluster level and the node level, but it doesn't work except at the VM/container level.
Whatever rules I make, I can bypass them like they didn't even exist


This is what I did:
1.
After and before enabling/adding rules to the node firewall,
pve-firewall restart && systemctl restart networking.service && systemctl restart pve-firewall.service
nothing changed

2.
Here is the output of
iptables-save

# Generated by iptables-save v1.8.9 on Mon Aug 28 19:01:03 2023
*raw
:PREROUTING ACCEPT [59313:97825222]
:OUTPUT ACCEPT [41395:42009319]
COMMIT
# Completed on Mon Aug 28 19:01:03 2023
# Generated by iptables-save v1.8.9 on Mon Aug 28 19:01:03 2023
*filter
:INPUT ACCEPT [259:16452]
:FORWARD ACCEPT [128:6760]
:OUTPUT ACCEPT [15:1140]
:PVEFW-Drop - [0:0]
:PVEFW-DropBroadcast - [0:0]
:PVEFW-FORWARD - [0:0]
:PVEFW-FWBR-IN - [0:0]
:PVEFW-FWBR-OUT - [0:0]
:PVEFW-HOST-IN - [0:0]
:PVEFW-HOST-OUT - [0:0]
:PVEFW-INPUT - [0:0]
:PVEFW-OUTPUT - [0:0]
:PVEFW-Reject - [0:0]
:PVEFW-SET-ACCEPT-MARK - [0:0]
:PVEFW-logflags - [0:0]
:PVEFW-reject - [0:0]
:PVEFW-smurflog - [0:0]
:PVEFW-smurfs - [0:0]
:PVEFW-tcpflags - [0:0]
:veth104i0-IN - [0:0]
:veth104i0-OUT - [0:0]
-A INPUT -j PVEFW-INPUT
-A FORWARD -j PVEFW-FORWARD
-A OUTPUT -j PVEFW-OUTPUT
-A PVEFW-Drop -j PVEFW-DropBroadcast
-A PVEFW-Drop -p icmp -m icmp --icmp-type 3/4 -j ACCEPT
-A PVEFW-Drop -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A PVEFW-Drop -m conntrack --ctstate INVALID -j DROP
-A PVEFW-Drop -p udp -m multiport --dports 135,445 -j DROP
-A PVEFW-Drop -p udp -m udp --dport 137:139 -j DROP
-A PVEFW-Drop -p udp -m udp --sport 137 --dport 1024:65535 -j DROP
-A PVEFW-Drop -p tcp -m multiport --dports 135,139,445 -j DROP
-A PVEFW-Drop -p udp -m udp --dport 1900 -j DROP
-A PVEFW-Drop -p tcp -m tcp ! --tcp-flags FIN,SYN,RST,ACK SYN -j DROP
-A PVEFW-Drop -p udp -m udp --sport 53 -j DROP
-A PVEFW-Drop -m comment --comment "PVESIG:83WlR/a4wLbmURFqMQT3uJSgIG8"
-A PVEFW-DropBroadcast -m addrtype --dst-type BROADCAST -j DROP
-A PVEFW-DropBroadcast -m addrtype --dst-type MULTICAST -j DROP
-A PVEFW-DropBroadcast -m addrtype --dst-type ANYCAST -j DROP
-A PVEFW-DropBroadcast -d 224.0.0.0/4 -j DROP
-A PVEFW-DropBroadcast -m comment --comment "PVESIG:NyjHNAtFbkH7WGLamPpdVnxHy4w"
-A PVEFW-FORWARD -m conntrack --ctstate INVALID -j DROP
-A PVEFW-FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A PVEFW-FORWARD -m physdev --physdev-in fwln+ --physdev-is-bridged -j PVEFW-FWBR-IN
-A PVEFW-FORWARD -m physdev --physdev-out fwln+ --physdev-is-bridged -j PVEFW-FWBR-OUT
-A PVEFW-FORWARD -m comment --comment "PVESIG:qnNexOcGa+y+jebd4dAUqFSp5nw"
-A PVEFW-FWBR-IN -m conntrack --ctstate INVALID,NEW -j PVEFW-smurfs
-A PVEFW-FWBR-IN -p tcp -j PVEFW-tcpflags
-A PVEFW-FWBR-IN -m physdev --physdev-out veth104i0 --physdev-is-bridged -j veth104i0-IN
-A PVEFW-FWBR-IN -m comment --comment "PVESIG:f0moHMJd/1oYEN/50PucSj2s05U"
-A PVEFW-FWBR-OUT -m physdev --physdev-in veth104i0 --physdev-is-bridged -j veth104i0-OUT
-A PVEFW-FWBR-OUT -m comment --comment "PVESIG:TpViXOJGjY2u196L2rZik0r1OmY"
-A PVEFW-HOST-IN -i lo -j ACCEPT
-A PVEFW-HOST-IN -m conntrack --ctstate INVALID -j DROP
-A PVEFW-HOST-IN -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A PVEFW-HOST-IN -m conntrack --ctstate INVALID,NEW -j PVEFW-smurfs
-A PVEFW-HOST-IN -p tcp -j PVEFW-tcpflags
-A PVEFW-HOST-IN -p igmp -j RETURN
-A PVEFW-HOST-IN -d 192.168.1.23/32 -i vmbr0 -j DROP
-A PVEFW-HOST-IN -s 192.168.1.0/24 -i vmbr0 -j RETURN
-A PVEFW-HOST-IN -d 192.168.1.0/24 -i vmbr0 -j DROP
-A PVEFW-HOST-IN -p tcp -m set --match-set PVEFW-0-management-v4 src -m tcp --dport 8006 -j RETURN
-A PVEFW-HOST-IN -p tcp -m set --match-set PVEFW-0-management-v4 src -m tcp --dport 5900:5999 -j RETURN
-A PVEFW-HOST-IN -p tcp -m set --match-set PVEFW-0-management-v4 src -m tcp --dport 3128 -j RETURN
-A PVEFW-HOST-IN -p tcp -m set --match-set PVEFW-0-management-v4 src -m tcp --dport 22 -j RETURN
-A PVEFW-HOST-IN -p tcp -m set --match-set PVEFW-0-management-v4 src -m tcp --dport 60000:60050 -j RETURN
-A PVEFW-HOST-IN -j PVEFW-Drop
-A PVEFW-HOST-IN -j DROP
-A PVEFW-HOST-IN -m comment --comment "PVESIG:mlBDA0UvyynvpVyZX4y0iVfhccs"
-A PVEFW-HOST-OUT -o lo -j ACCEPT
-A PVEFW-HOST-OUT -m conntrack --ctstate INVALID -j DROP
-A PVEFW-HOST-OUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A PVEFW-HOST-OUT -p igmp -j RETURN
-A PVEFW-HOST-OUT -o vmbr0 -j RETURN
-A PVEFW-HOST-OUT -d 192.168.1.0/24 -p tcp -m tcp --dport 8006 -j RETURN
-A PVEFW-HOST-OUT -d 192.168.1.0/24 -p tcp -m tcp --dport 22 -j RETURN
-A PVEFW-HOST-OUT -d 192.168.1.0/24 -p tcp -m tcp --dport 5900:5999 -j RETURN
-A PVEFW-HOST-OUT -d 192.168.1.0/24 -p tcp -m tcp --dport 3128 -j RETURN
-A PVEFW-HOST-OUT -j RETURN
-A PVEFW-HOST-OUT -m comment --comment "PVESIG:Qqu08QqSmQP68M3P6vTBiOlhuz0"
-A PVEFW-INPUT -j PVEFW-HOST-IN
-A PVEFW-INPUT -m comment --comment "PVESIG:+5iMmLaxKXynOB/+5xibfx7WhFk"
-A PVEFW-OUTPUT -j PVEFW-HOST-OUT
-A PVEFW-OUTPUT -m comment --comment "PVESIG:LjHoZeSSiWAG3+2ZAyL/xuEehd0"
-A PVEFW-Reject -j PVEFW-DropBroadcast
-A PVEFW-Reject -p icmp -m icmp --icmp-type 3/4 -j ACCEPT
-A PVEFW-Reject -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A PVEFW-Reject -m conntrack --ctstate INVALID -j DROP
-A PVEFW-Reject -p udp -m multiport --dports 135,445 -j PVEFW-reject
-A PVEFW-Reject -p udp -m udp --dport 137:139 -j PVEFW-reject
-A PVEFW-Reject -p udp -m udp --sport 137 --dport 1024:65535 -j PVEFW-reject
-A PVEFW-Reject -p tcp -m multiport --dports 135,139,445 -j PVEFW-reject
-A PVEFW-Reject -p udp -m udp --dport 1900 -j DROP
-A PVEFW-Reject -p tcp -m tcp ! --tcp-flags FIN,SYN,RST,ACK SYN -j DROP
-A PVEFW-Reject -p udp -m udp --sport 53 -j DROP
-A PVEFW-Reject -m comment --comment "PVESIG:h3DyALVslgH5hutETfixGP08w7c"
-A PVEFW-SET-ACCEPT-MARK -j MARK --set-xmark 0x80000000/0x80000000
-A PVEFW-SET-ACCEPT-MARK -m comment --comment "PVESIG:Hg/OIgIwJChBUcWU8Xnjhdd2jUY"
-A PVEFW-logflags -j DROP
-A PVEFW-logflags -m comment --comment "PVESIG:MN4PH1oPZeABMuWr64RrygPfW7A"
-A PVEFW-reject -m addrtype --dst-type BROADCAST -j DROP
-A PVEFW-reject -s 224.0.0.0/4 -j DROP
-A PVEFW-reject -p icmp -j DROP
-A PVEFW-reject -p tcp -j REJECT --reject-with tcp-reset
-A PVEFW-reject -p udp -j REJECT --reject-with icmp-port-unreachable
-A PVEFW-reject -p icmp -j REJECT --reject-with icmp-host-unreachable
-A PVEFW-reject -j REJECT --reject-with icmp-host-prohibited
-A PVEFW-reject -m comment --comment "PVESIG:Jlkrtle1mDdtxDeI9QaDSL++Npc"
-A PVEFW-smurflog -j DROP
-A PVEFW-smurflog -m comment --comment "PVESIG:2gfT1VMkfr0JL6OccRXTGXo+1qk"
-A PVEFW-smurfs -s 0.0.0.0/32 -j RETURN
-A PVEFW-smurfs -m addrtype --src-type BROADCAST -g PVEFW-smurflog
-A PVEFW-smurfs -s 224.0.0.0/4 -g PVEFW-smurflog
-A PVEFW-smurfs -m comment --comment "PVESIG:HssVe5QCBXd5mc9kC88749+7fag"
-A PVEFW-tcpflags -p tcp -m tcp --tcp-flags FIN,SYN,RST,PSH,ACK,URG FIN,PSH,URG -g PVEFW-logflags
-A PVEFW-tcpflags -p tcp -m tcp --tcp-flags FIN,SYN,RST,PSH,ACK,URG NONE -g PVEFW-logflags
-A PVEFW-tcpflags -p tcp -m tcp --tcp-flags SYN,RST SYN,RST -g PVEFW-logflags
-A PVEFW-tcpflags -p tcp -m tcp --tcp-flags FIN,SYN FIN,SYN -g PVEFW-logflags
-A PVEFW-tcpflags -p tcp -m tcp --sport 0 --tcp-flags FIN,SYN,RST,ACK SYN -g PVEFW-logflags
-A PVEFW-tcpflags -m comment --comment "PVESIG:CMFojwNPqllyqD67NeI5m+bP5mo"
-A veth104i0-IN -p udp -m udp --sport 67 --dport 68 -j ACCEPT
-A veth104i0-IN -j ACCEPT
-A veth104i0-IN -m comment --comment "PVESIG:oRIvPdmGhjgdA/jX4tLb7d0QD7E"
-A veth104i0-OUT -p udp -m udp --sport 68 --dport 67 -g PVEFW-SET-ACCEPT-MARK
-A veth104i0-OUT -m mac ! --mac-source 16:a5:89:b2:c6:94 -j DROP
-A veth104i0-OUT -j MARK --set-xmark 0x0/0x80000000
-A veth104i0-OUT -g PVEFW-SET-ACCEPT-MARK
-A veth104i0-OUT -m comment --comment "PVESIG:ETlCEsuhoce37e675QKSFNLJaQo"
COMMIT
# Completed on Mon Aug 28 19:01:03 2023

3. Rebooting, still the same

4.Installing CEPH, still the same from https://forum.proxmox.com/threads/pve-firewall-not-filtering-anything.67084/page-2

5. enabling TCP flags filter. still the same from https://www.reddit.com/r/Proxmox/comments/13m8ccj/firewall_not_working_on_proxmox_743_help/

More infos:
$pve-firewall status

State: Enabled/Running

from https://forum.proxmox.com/threads/pve-firewall-not-filtering-anything.67084/page-1
$sysctl -a |grep call

net.bridge.bridge-nf-call-arptables = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1

cat /etc/pve/firewall/cluster.fw

[OPTIONS]

enable: 1

[RULES]

|IN ACCEPT -i vmbr0 -dest 192.168.1.23/32 -log nolog
|IN REJECT -i vmbr0 -p tcp -dport 5005 -log nolog
|IN ACCEPT -i vmbr0 -source 192.168.1.0/24 -p tcp -dport 8006 -log nolog
|IN ACCEPT -i vmbr0 -source 192.168.1.0/24 -p tcp -dport 8006 -log nolog

$cat /etc/pve/nodes/pve/host.fw

[OPTIONS]

tcpflags: 1
enable: 1

[RULES]

IN DROP -i vmbr0 -dest 192.168.1.23/32 -log nolog # For test purposes
OUT ACCEPT -i vmbr0 -log nolog # Traffic sortant autorisé
IN ACCEPT -i vmbr0 -source 192.168.1.0/24 -log nolog # Traffic entrant dans le reseau local autorise
IN DROP -i vmbr0 -dest 192.168.1.0/24 -log nolog

cat /etc/network/interfaces

auto lo
iface lo inet loopback

iface enp0s31f6 inet manual

auto vmbr0
iface vmbr0 inet static
        address 192.168.1.88/24
        gateway 192.168.1.1
        bridge-ports enp0s31f6
        bridge-stp off
        bridge-fd 0

Here are some screenshots of the rules I tried to make
At cluster level

At node level

I know that some of these don't make any sense, but they should still have an effect.

Could anyone help me out?
Thanks in advance.

Sorry for my english

 

[m0n0pr1x]

I actually resolved the issue. I feel so bad now. I should have RTFM. I didn't know that VMs didn't inherit their firewall settings from their node. Each VM/container must be configured individually. Only the node inherits from its datacenter firewall rules.