My Truenas core vm is crashing when I try to pass my HBA to it. Both Intel Virtualization Technology and VT-d is enabled in the bios.
I've gone through all the whole of 10.9: https://pve.proxmox.com/pve-docs/pve-admin-guide.html#qm_pci_passthrough
and here are the changes I've made as per 10.9:
and this is the error on start
Any help would be great thank you.
I've gone through all the whole of 10.9: https://pve.proxmox.com/pve-docs/pve-admin-guide.html#qm_pci_passthrough
Bash:
root@BarryProxmox:~# pveversion -v
proxmox-ve: 7.2-1 (running kernel: 5.15.30-2-pve)
pve-manager: 7.2-3 (running version: 7.2-3/c743d6c1)
pve-kernel-helper: 7.2-2
pve-kernel-5.15: 7.2-1
pve-kernel-5.15.30-2-pve: 5.15.30-3
ceph-fuse: 15.2.16-pve1
corosync: 3.1.5-pve2
criu: 3.15-1+pve-1
glusterfs-client: 9.2-1
ifupdown2: 3.1.0-1+pmx3
ksm-control-daemon: 1.4-1
libjs-extjs: 7.0.0-1
libknet1: 1.22-pve2
libproxmox-acme-perl: 1.4.2
libproxmox-backup-qemu0: 1.2.0-1
libpve-access-control: 7.1-8
libpve-apiclient-perl: 3.2-1
libpve-common-perl: 7.1-6
libpve-guest-common-perl: 4.1-2
libpve-http-server-perl: 4.1-1
libpve-storage-perl: 7.2-2
libspice-server1: 0.14.3-2.1
lvm2: 2.03.11-2.1
lxc-pve: 4.0.12-1
lxcfs: 4.0.12-pve1
novnc-pve: 1.3.0-3
proxmox-backup-client: 2.1.8-1
proxmox-backup-file-restore: 2.1.8-1
proxmox-mini-journalreader: 1.3-1
proxmox-widget-toolkit: 3.4-10
pve-cluster: 7.2-1
pve-container: 4.2-1
pve-docs: 7.2-2
pve-edk2-firmware: 3.20210831-2
pve-firewall: 4.2-5
pve-firmware: 3.4-1
pve-ha-manager: 3.3-4
pve-i18n: 2.7-1
pve-qemu-kvm: 6.2.0-5
pve-xtermjs: 4.16.0-1
qemu-server: 7.2-2
smartmontools: 7.2-pve3
spiceterm: 3.2-2
swtpm: 0.7.1~bpo11+1
vncterm: 1.7-1
zfsutils-linux: 2.1.4-pve1
root@BarryProxmox:~# qm config 101
acpi: 1
boot: order=sata0;ide2;net0
cores: 4
efidisk0: local-lvm:vm-101-disk-0,efitype=4m,pre-enrolled-keys=1,size=4M
hostpci0: 0000:01:00,pcie=1,rombar=0
ide2: none,media=cdrom
machine: q35
memory: 8192
meta: creation-qemu=6.2.0,ctime=1657289462
name: BarryTruenasCore
net0: virtio=96:E8:4B:91:3E:0B,bridge=vmbr0,firewall=1
numa: 0
ostype: l26
sata0: local-lvm:vm-101-disk-1,size=32G,ssd=1
scsihw: virtio-scsi-pci
smbios1: uuid=8761a41e-caf6-4de0-ad08-126a0778e03a
sockets: 1
vmgenid: 403d6eaf-185b-4c6a-bedc-cc439685f462
root@BarryProxmox:~# free -h
total used free shared buff/cache available
Mem: 7.5Gi 1.1Gi 6.1Gi 67Mi 321Mi 6.1Gi
Swap: 7.0Gi 0B 7.0Gi
root@BarryProxmox:~# qm showcmd 101 --pretty
/usr/bin/kvm \
-id 101 \
-name BarryTruenasCore \
-no-shutdown \
-chardev 'socket,id=qmp,path=/var/run/qemu-server/101.qmp,server=on,wait=off' \
-mon 'chardev=qmp,mode=control' \
-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
-mon 'chardev=qmp-event,mode=control' \
-pidfile /var/run/qemu-server/101.pid \
-daemonize \
-smbios 'type=1,uuid=8761a41e-caf6-4de0-ad08-126a0778e03a' \
-smp '4,sockets=1,cores=4,maxcpus=4' \
-nodefaults \
-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
-vnc 'unix:/var/run/qemu-server/101.vnc,password=on' \
-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
-m 8192 \
-readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
-device 'vmgenid,guid=403d6eaf-185b-4c6a-bedc-cc439685f462' \
-device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
-device 'vfio-pci,host=0000:01:00.0,id=hostpci0.0,bus=ich9-pcie-port-1,addr=0x0.0,rombar=0,multifunction=on' \
-device 'vfio-pci,host=0000:01:00.1,id=hostpci0.1,bus=ich9-pcie-port-1,addr=0x0.1' \
-device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
-iscsi 'initiator-name=iqn.1993-08.org.debian:01:9350b96b3dfd' \
-drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=101' \
-device 'ahci,id=ahci0,multifunction=on,bus=pci.0,addr=0x7' \
-drive 'file=/dev/pve/vm-101-disk-1,if=none,id=drive-sata0,format=raw,cache=none,aio=io_uring,detect-zeroes=on' \
-device 'ide-hd,bus=ahci0.0,drive=drive-sata0,id=sata0,rotation_rate=1,bootindex=100' \
-netdev 'type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' \
-device 'virtio-net-pci,mac=96:E8:4B:91:3E:0B,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=102' \
-machine 'type=q35+pve0'
root@BarryProxmox:~#
and here are the changes I've made as per 10.9:
Bash:
root@BarryProxmox:~# cat /etc/modules
# /etc/modules: kernel modules to load at boot time.
#
# This file contains the names of kernel modules that should be loaded
# at boot time, one per line. Lines beginning with "#" are ignored.
vfio
vfio_iommu_type1
vfio_pci
vfio_virqfd
root@BarryProxmox:~# dmesg | grep -e DMAR -e IOMMU -e AMD-Vi
[ 0.008417] ACPI: DMAR 0x000000007ED6F1C8 0000A8 (v01 INTEL KBL 00000001 INTL 00000001)
[ 0.008439] ACPI: Reserving DMAR table memory at [mem 0x7ed6f1c8-0x7ed6f26f]
[ 0.019295] DMAR: IOMMU enabled
[ 0.058483] DMAR: Host address width 39
[ 0.058484] DMAR: DRHD base: 0x000000fed90000 flags: 0x0
[ 0.058488] DMAR: dmar0: reg_base_addr fed90000 ver 1:0 cap 1c0000c40660462 ecap 19e2ff0505e
[ 0.058490] DMAR: DRHD base: 0x000000fed91000 flags: 0x1
[ 0.058493] DMAR: dmar1: reg_base_addr fed91000 ver 1:0 cap d2008c40660462 ecap f050da
[ 0.058495] DMAR: RMRR base: 0x0000007e52e000 end: 0x0000007e54dfff
[ 0.058497] DMAR: RMRR base: 0x0000007f800000 end: 0x0000008fffffff
[ 0.058498] DMAR-IR: IOAPIC id 2 under DRHD base 0xfed91000 IOMMU 1
[ 0.058500] DMAR-IR: HPET id 0 under DRHD base 0xfed91000
[ 0.058501] DMAR-IR: Queued invalidation will be enabled to support x2apic and Intr-remapping.
[ 0.060012] DMAR-IR: Enabled IRQ remapping in x2apic mode
[ 0.225393] DMAR: No ATSR found
[ 0.225394] DMAR: No SATC found
[ 0.225395] DMAR: IOMMU feature fl1gp_support inconsistent
[ 0.225396] DMAR: IOMMU feature pgsel_inv inconsistent
[ 0.225397] DMAR: IOMMU feature nwfs inconsistent
[ 0.225398] DMAR: IOMMU feature pasid inconsistent
[ 0.225399] DMAR: IOMMU feature eafs inconsistent
[ 0.225399] DMAR: IOMMU feature prs inconsistent
[ 0.225400] DMAR: IOMMU feature nest inconsistent
[ 0.225400] DMAR: IOMMU feature mts inconsistent
[ 0.225401] DMAR: IOMMU feature sc_support inconsistent
[ 0.225401] DMAR: IOMMU feature dev_iotlb_support inconsistent
[ 0.225402] DMAR: dmar0: Using Queued invalidation
[ 0.225405] DMAR: dmar1: Using Queued invalidation
[ 0.225744] DMAR: Intel(R) Virtualization Technology for Directed I/O
root@BarryProxmox:~# cat /proc/cmdline
BOOT_IMAGE=/boot/vmlinuz-5.15.30-2-pve root=/dev/mapper/pve-root ro quiet intel_iommu=on iommu=pt
root@BarryProxmox:~# lspci -nnk
..........
..........
01:00.0 Fibre Channel [0c04]: Emulex Corporation LPe12000 Series 8Gb Fibre Channel Adapter [10df:f100] (rev 03)
Subsystem: Emulex Corporation LPe12000 Series 8Gb Fibre Channel Adapter [10df:f100]
Kernel driver in use: vfio-pci
Kernel modules: lpfc
01:00.1 Fibre Channel [0c04]: Emulex Corporation LPe12000 Series 8Gb Fibre Channel Adapter [10df:f100] (rev 03)
Subsystem: Emulex Corporation LPe12000 Series 8Gb Fibre Channel Adapter [10df:f100]
Kernel driver in use: vfio-pci
Kernel modules: lpfc
root@BarryProxmox:~#
Bash:
root@BarryProxmox:~# qm start 101
start failed: command '/usr/bin/kvm -id 101 -name BarryTruenasCore -no-shutdown -chardev 'socket,id=qmp,path=/var/run/qemu-server/101.qmp,server=on,wait=off' -mon 'chardev=qmp,mode=control' -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' -mon 'chardev=qmp-event,mode=control' -pidfile /var/run/qemu-server/101.pid -daemonize -smbios 'type=1,uuid=8761a41e-caf6-4de0-ad08-126a0778e03a' -smp '4,sockets=1,cores=4,maxcpus=4' -nodefaults -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' -vnc 'unix:/var/run/qemu-server/101.vnc,password=on' -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep -m 8192 -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg -device 'vmgenid,guid=403d6eaf-185b-4c6a-bedc-cc439685f462' -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' -device 'vfio-pci,host=0000:01:00.0,id=hostpci0.0,bus=ich9-pcie-port-1,addr=0x0.0,rombar=0,multifunction=on' -device 'vfio-pci,host=0000:01:00.1,id=hostpci0.1,bus=ich9-pcie-port-1,addr=0x0.1' -device 'VGA,id=vga,bus=pcie.0,addr=0x1' -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' -iscsi 'initiator-name=iqn.1993-08.org.debian:01:9350b96b3dfd' -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=101' -device 'ahci,id=ahci0,multifunction=on,bus=pci.0,addr=0x7' -drive 'file=/dev/pve/vm-101-disk-1,if=none,id=drive-sata0,format=raw,cache=none,aio=io_uring,detect-zeroes=on' -device 'ide-hd,bus=ahci0.0,drive=drive-sata0,id=sata0,rotation_rate=1,bootindex=100' -netdev 'type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=96:E8:4B:91:3E:0B,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=102' -machine 'type=q35+pve0'' failed: got timeout
Any help would be great thank you.
Last edited: