Hi everyone,
Having some trouble and I am not sure if its just not possible (as my networking is not the best)
Background
3 node cluster each node has a 1GbE interface and a 10GbE SFP+ interface. Cluster works just fine.
Each cluster looks like this (network) and migration, storage, management is okay
I have then defined 3 SDN networks (all are running okay or at least appear to be)
And finally on the host where eth0 is the k8s and enp6s19 is the storage (10GbE)
I can route anywhere except VLAN 110 (Destination unreachable).
On the PVE host no issues on 110. So the question becomes can I actually have a VLAN on the base PVE 110 assigned and working and then re-use that same VLAN in SDN to pass into hosts to utilise the storage network (VLAN 110) to gain access to the 10GbE for NFS mounts etc?
Have I fundamentally misunderstood something, if so pleae let me know where have i gone wrong. I feel like I have tried everything.
I was also able to confirm on the Ubiquiti UDM side that it can see the device and any IP changes against the MAC address so can only assume I have made an error in config.
Having some trouble and I am not sure if its just not possible (as my networking is not the best)
Background
3 node cluster each node has a 1GbE interface and a 10GbE SFP+ interface. Cluster works just fine.
Each cluster looks like this (network) and migration, storage, management is okay
YAML:
auto lo
iface lo inet loopback
auto eno1
iface eno1 inet manual
auto enp1s0f0
iface enp1s0f0 inet manual
pre-up ethtool -G enp1s0f1 tx 4096 rx 8192
#10G Network
iface enp1s0f1 inet manual
auto enp1s0f0.202
iface enp1s0f0.202 inet static
address 192.168.202.2/29
#Migration
auto eno1.201
iface eno1.201 inet static
address 192.168.201.2/29
#Cluster
auto enp1s0f0.110
iface enp1s0f0.110 inet static
address 192.168.110.2/24
#Storage Network
auto vmbr0
iface vmbr0 inet manual
bridge-ports eno1
bridge-stp off
bridge-fd 0
bridge-vlan-aware yes
bridge-vids 2-4094
#Services Network
auto vmbr0.203
iface vmbr0.203 inet static
address 192.168.203.2/29
gateway 192.168.203.1
#Management
auto vmbr1
iface vmbr1 inet manual
bridge-ports enp1s0f0
bridge-stp off
bridge-fd 0
bridge-vlan-aware yes
bridge-vids 2-4094
#Storage
source /etc/network/interfaces.d/*I have then defined 3 SDN networks (all are running okay or at least appear to be)
YAML:
root@pve01:/etc/pve/sdn# cat subnets.cfg
subnet: internal-192.168.220.0-24
vnet internal
gateway 192.168.220.1
snat 1
subnet: k8s-192.168.150.0-24
vnet k8s
gateway 192.168.150.1
snat 1
subnet: storage-192.168.110.0-24
vnet storage
gateway 192.168.110.1
root@pve01:/etc/pve/sdn# cat zones.cfg
vlan: internal
bridge vmbr0
ipam pve
vlan: k8s
bridge vmbr0
ipam pve
vlan: storage
bridge vmbr1
ipam pve
root@pve01:/etc/pve/sdn# cat vnets.cfg
vnet: internal
zone internal
alias internal-services
tag 220
vnet: k8s
zone k8s
alias Kubernetes Services
tag 150
vnet: storage
zone storage
alias Storage Network for Services
tag 110And finally on the host where eth0 is the k8s and enp6s19 is the storage (10GbE)
YAML:
network:
version: 2
ethernets:
eth0:
match:
macaddress: "02:f4:25:15:27:0b"
addresses:
- "192.168.150.2/24"
nameservers:
addresses:
- 192.168.150.1
search:
- xxx.net
set-name: "eth0"
routes:
- to: "default"
via: 192.168.150.1
- to: 192.168.150.0/24
via: 192.168.150.1
table: 101
routing-policy:
- from: 192.168.150.0/24
table: 101
enp6s19:
addresses:
- "192.168.110.110/24"
nameservers:
addresses:
- 192.168.110.1
search:
- xxx.net
routes:
- to: 192.168.110.0/24
via: 192.168.110.1
table: 102
routing-policy:
- from: 192.168.110.0/24
table: 102
YAML:
root@rancher0101:/etc/netplan# ip route list
default via 192.168.150.1 dev eth0 proto static
192.168.110.0/24 dev enp6s19 proto kernel scope link src 192.168.110.110
192.168.150.0/24 dev eth0 proto kernel scope link src 192.168.150.2I can route anywhere except VLAN 110 (Destination unreachable).
On the PVE host no issues on 110. So the question becomes can I actually have a VLAN on the base PVE 110 assigned and working and then re-use that same VLAN in SDN to pass into hosts to utilise the storage network (VLAN 110) to gain access to the 10GbE for NFS mounts etc?
Have I fundamentally misunderstood something, if so pleae let me know where have i gone wrong. I feel like I have tried everything.
I was also able to confirm on the Ubiquiti UDM side that it can see the device and any IP changes against the MAC address so can only assume I have made an error in config.
