I have a vm with an hba doing pci passthrough to truenas. I started having issues after i added a second hba with an identical chipset, but did not want passthrough for the 2nd hba, i did some trickery binding the bus address to use the vfio driver per this post, i created the referenced vfio_bind script in the post linked within the post:
https://forum.proxmox.com/threads/p...h-have-the-same-device-id.117542/#post-508720
if i remove the pci passthrough config the vm boots fine. I managed to get it boot once with the pci passthrough enabled by trying to commands i read using the cli and doing a set lock suspend, but it seems to have only worked once. i can't seem to replicate it.
vm config
relevant output from lspci -v
trying to start vm from cli
https://forum.proxmox.com/threads/p...h-have-the-same-device-id.117542/#post-508720
if i remove the pci passthrough config the vm boots fine. I managed to get it boot once with the pci passthrough enabled by trying to commands i read using the cli and doing a set lock suspend, but it seems to have only worked once. i can't seem to replicate it.
vm config
Code:
root@pve1:~# qm config 101
agent: 1
balloon: 0
bios: ovmf
boot: order=scsi0;ide2;net0
cores: 4
cpu: host
efidisk0: local-zfs:vm-101-disk-0,efitype=4m,pre-enrolled-keys=1,size=1M
hostpci0: 0000:03:00,pcie=1
ide2: none,media=cdrom
machine: q35
memory: 32736
meta: creation-qemu=6.2.0,ctime=1667261155
name: TrueNAS
net0: virtio=C2:66:C1:B4:B1:81,bridge=vmbr1
net1: virtio=7E:61:2A:D5:18:D9,bridge=vmbr1,tag=110
numa: 0
ostype: other
scsi0: local-zfs:vm-101-disk-1,size=32G
scsihw: virtio-scsi-pci
smbios1: uuid=77a815e6-1c6d-48bb-862f-cae8d0c4cccf
sockets: 1
vga: qxl
vmgenid: 66490629-97ed-4d4c-bf0a-ce60c47f18ddrelevant output from lspci -v
Code:
0000:03:00.0 Serial Attached SCSI controller [0107]: Broadcom / LSI SAS3008 PCI-Express Fusion-MPT SAS-3 [1000:0097] (rev 02)
DeviceName: Integrated RAID
Subsystem: Dell HBA330 Mini [1028:1f53]
Flags: fast devsel, IRQ 329, NUMA node 0, IOMMU group 29
I/O ports at 2000 [size=256]
Memory at b5700000 (64-bit, non-prefetchable) [size=64K]
Memory at b4600000 (64-bit, non-prefetchable) [size=1M]
Expansion ROM at <ignored> [disabled]
Capabilities: [50] Power Management version 3
Capabilities: [68] Express Endpoint, MSI 00
Capabilities: [a8] MSI: Enable- Count=1/1 Maskable+ 64bit+
Capabilities: [c0] MSI-X: Enable- Count=96 Masked-
Capabilities: [100] Advanced Error Reporting
Capabilities: [1e0] Secondary PCI Express
Capabilities: [1c0] Power Budgeting <?>
Capabilities: [190] Dynamic Power Allocation <?>
Capabilities: [150] Single Root I/O Virtualization (SR-IOV)
Capabilities: [148] Alternative Routing-ID Interpretation (ARI)
Kernel driver in use: vfio-pci
Kernel modules: mpt3sas
0000:84:00.0 Serial Attached SCSI controller [0107]: Broadcom / LSI SAS3008 PCI-Express Fusion-MPT SAS-3 [1000:0097] (rev 02)
Subsystem: Broadcom / LSI SAS9300-8i [1000:30e0]
Flags: bus master, fast devsel, latency 0, IRQ 41, NUMA node 1, IOMMU group 83
I/O ports at 8000 [size=256]
Memory at c8040000 (64-bit, non-prefetchable) [size=64K]
Memory at c8000000 (64-bit, non-prefetchable) [size=256K]
Expansion ROM at <ignored> [disabled]
Capabilities: [50] Power Management version 3
Capabilities: [68] Express Endpoint, MSI 00
Capabilities: [a8] MSI: Enable- Count=1/1 Maskable+ 64bit+
Capabilities: [c0] MSI-X: Enable+ Count=96 Masked-
Capabilities: [100] Advanced Error Reporting
Capabilities: [1e0] Secondary PCI Express
Capabilities: [1c0] Power Budgeting <?>
Capabilities: [190] Dynamic Power Allocation <?>
Capabilities: [148] Alternative Routing-ID Interpretation (ARI)
Kernel driver in use: mpt3sas
Kernel modules: mpt3sastrying to start vm from cli
Code:
root@pve1:~# qm start 101
start failed: command '/usr/bin/kvm -id 101 -name TrueNAS -no-shutdown -chardev 'socket,id=qmp,path=/var/run/qemu-server/101.qmp,server=on,wait=off' -mon 'chardev=qmp,mode=control' -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' -mon 'chardev=qmp-event,mode=control' -pidfile /var/run/qemu-server/101.pid -daemonize -smbios 'type=1,uuid=77a815e6-1c6d-48bb-862f-cae8d0c4cccf' -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.secboot.fd' -drive 'if=pflash,unit=1,format=raw,id=drive-efidisk0,size=540672,file=/dev/zvol/rpool/data/vm-101-disk-0' -smp '4,sockets=1,cores=4,maxcpus=4' -nodefaults -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' -vnc 'unix:/var/run/qemu-server/101.vnc,password=on' -cpu host,+kvm_pv_eoi,+kvm_pv_unhalt -m 32736 -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg -device 'vmgenid,guid=66490629-97ed-4d4c-bf0a-ce60c47f18dd' -device 'vfio-pci,host=0000:03:00.0,id=hostpci0,bus=ich9-pcie-port-1,addr=0x0' -device 'qxl-vga,id=vga,max_outputs=4,bus=pcie.0,addr=0x1' -chardev 'socket,path=/var/run/qemu-server/101.qga,server=on,wait=off,id=qga0' -device 'virtio-serial,id=qga0,bus=pci.0,addr=0x8' -device 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0' -device 'virtio-serial,id=spice,bus=pci.0,addr=0x9' -chardev 'spicevmc,id=vdagent,name=vdagent' -device 'virtserialport,chardev=vdagent,name=com.redhat.spice.0' -spice 'tls-port=61002,addr=127.0.0.1,tls-ciphers=HIGH,seamless-migration=on' -iscsi 'initiator-name=iqn.1993-08.org.debian:01:831f5732a8bf' -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=101' -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' -drive 'file=/dev/zvol/rpool/data/vm-101-disk-1,if=none,id=drive-scsi0,format=raw,cache=none,aio=io_uring,detect-zeroes=on' -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' -netdev 'type=tap,id=net0,ifname=tap101i0,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=C2:66:C1:B4:B1:81,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=102' -netdev 'type=tap,id=net1,ifname=tap101i1,script=/var/lib/qemu-server/pve-bridge,downscript=/var/lib/qemu-server/pve-bridgedown,vhost=on' -device 'virtio-net-pci,mac=7E:61:2A:D5:18:D9,netdev=net1,bus=pci.0,addr=0x13,id=net1' -machine 'type=q35+pve0'' failed: got timeout
Last edited:
