BSOD in Windows 11 with GPU passthorugh & nested virtualisation

Aug 7, 2023
26
2
3
Hi Team

I have tried everything from fresh install to different motherboard, i am just not able to understand why Windows 11 video_dxgkrnl_fatal_error happening with GPU passthrough and Nested WSL2

Issue: Windows 11 BSOD every 11 min or in reboot/shutdown


My Proxmox details:

pve-manager/8.1.4/ec5affc9e41f1d79 (running kernel: 6.5.11-8-pve)

Global C state is off, No issues with Memory or SSD, Proper cooling and fitted GPU

Mobo - X670E Creator ProArt
GPU: 4060
CPU: AMD Ryzen 9 7900X 12-Core

All MSI interrupt enabled and everything enabled.

Tried on : A2000/A4000 & Other Motherboards as well


Things i have tried:

1. -cpu 'host,+svm' or -cpu 'host' or -cpu 'host,hypervisor=off'
2. Fresh DDU based install of GPU driver with Fast boot off and MSI Interrupt High
3. Tweaked all host settings on MSI
4. Switch CPU model to EPYC 3rd gen etc



What works (one way):

1. setting cpu to qemu/kvm (No CRASH but nested virtualisation doesn't work)

2. Disable GPU/NVIDIA Driver (No CRASH, nested virtualisation works but no GPU)


Please help me with this.
 
Last edited:
Also @MarkusF i checked this

  • Corrupt hardware
  • Outdated or corrupt graphics or display driver
  • Disk with bad sector or corrupt firmware
  • Not enough storage space to run processes.
  • Changes made to the BIOS.
  • Security application causing interference.
  • Corrupt DLL and Windows System Files.
  • Virus Attack.
  • Incorrect installation of DirectX.
As it was full fresh installation, i have tried on 3 different servers (freshly build and tested in terms of Ram, PSU and SSD) with proper cooling and latest drivers (studio and professional and geforce edition) and BIOS are also nothing changed but i disabled Global State also tried 4G Decoding off but still no luck
 
I formated to Ubuntu KVM 22 basic and tried and it works fine although the kernal is 5.15.0-102-generic

Here is the stuff

kvm -cpu host,migratable=on,topoext=on,svm=off,amd-stibp=off,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vpindex=on,hv-runtime=on,hv-synic=on,hv-stimer=on,hv-reset=on,hv-vendor-id=123456789ab,hv-frequencies=on,kvm=off -m 16384 -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":17179869184} -overcommit mem-lock=off -smp 4,sockets=1,dies=1,cores=2,threads=2 -uuid c39818be-3567-41f3-8c1b-b9a511cbe353 -smbios type=0,vendor=American Megatrends Inc.,version=1807,date=12/14/2023,release=18.7 -smbios type=1,manufacturer=ASUS,product=System Product Name,version=System Version,serial=System Serial Number,sku=SKU,family=To be filled by O.E.M. -smbios type=2,manufacturer=ASUSTeK COMPUTER INC.,product=ProArt B650-CREATOR,version=Rev 1.xx,serial=221213838100080,asset=Default string,location=Default string -smbios type=3,manufacturer=Default string,version=Default string,serial=Default string,asset=Default string,sku=Default string -smbios type=11,value=Default string,value=Default string,value=IKNOW,value=Default string,value=Default string,value=Default string -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=32,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=localtime,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=22,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=23,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device pcie-root-port,port=24,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3 -device pcie-root-port,port=25,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 -device pcie-root-port,port=26,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 -device pcie-root-port,port=27,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 -device pcie-root-port,port=28,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 -device pcie-root-port,port=29,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 -device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/Gaming.qcow2","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":false,"discard":"unmap","driver":"qcow2","file":"libvirt-1-storage","backing":null} -device ide-hd,bus=ide.0,drive=libvirt-1-format,id=sata0-0-0,bootindex=1 -netdev tap,fd=33,id=hostnet0 -device e1000e,netdev=hostnet0,id=net0,mac=52:54:00:5d:26:da,bus=pci.1,addr=0x0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev spicevmc,id=charchannel0,name=vdagent -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -audiodev {"id":"audio1","driver":"spice"} -vnc 0.0.0.0:0,audiodev=audio1 -device vmware-svga,id=video0,vgamem_mb=16,bus=pcie.0,addr=0x1 -device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 -chardev spicevmc,id=charredir0,name=usbredir -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev spicevmc,id=charredir1,name=usbredir -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device vfio-pci,host=0000:01:00.1,id=hostdev0,bus=pci.5,addr=0x0 -device vfio-pci,host=0000:01:00.0,id=hostdev1,bus=pci.6,addr=0x0 -device virtio-balloon-pci,id=balloon0,bus=pci.4,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on


<cputune>
<vcpupin vcpu='0' cpuset='1'/>
<vcpupin vcpu='1' cpuset='9'/>
<vcpupin vcpu='2' cpuset='2'/>
<vcpupin vcpu='3' cpuset='10'/>
</cputune>
<os>
<type arch='x86_64' machine='pc-q35-6.2'>hvm</type>
<boot dev='hd'/>
<smbios mode='host'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode='custom'>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<runtime state='on'/>
<synic state='on'/>
<stimer state='on'/>
<reset state='on'/>
<vendor_id state='on' value='123456789ab'/>
<frequencies state='on'/>
</hyperv>
<kvm>
<hidden state='on'/>
</kvm>
<vmport state='off'/>
<ioapic driver='kvm'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<topology sockets='1' dies='1' cores='2' threads='2'/>
<feature policy='require' name='topoext'/>
<feature policy='disable' name='svm'/>
<feature policy='disable' name='amd-stibp'/>
</cpu>


@MarkusF any suggestion what is wrong

BIOS and all settings is same i just reformatted to Ubuntu 22 and tried and it works no BSOD at all i ran for 12 hours
 
Last edited:
Hello,

maybe try some older or newer pve-kernels.

You could also try to set the same cpu parameter manually with args in the pve vm config (/etc/pve/qemu-server/):

Code:
args: -cpu host,migratable=on,topoext=on,svm=off,amd-stibp=off,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff,hv-vpindex=on,hv-runtime=on,hv-synic=on,hv-stimer=on,hv-reset=on,hv-vendor-id=123456789ab,hv-frequencies=on,kvm=off
 
Strange does not work on proxmox 7.4 with 5.15 but works fine with Ubuntu 22 with kernal 6.5.0-1019-oem

I removed all config/cpu flags from KVM xml and still no crash no BSOD for windows, works like a charm @MarkusF

Super super confused, whats going wrong on Proxmox vs Ubuntu/KVM just works fine

x86-64-v4 cpu works fine but host passthrough with migration hypervisor is not at all working :(

I tried disabling or manualy adding every cpu flag but still same





kvm -cpu host,migratable=on,hv-time=on,hv-relaxed=on,hv-vapic=on,hv-spinlocks=0x1fff -m 16384 -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":17179869184} -overcommit mem-lock=off -smp 4,sockets=1,dies=1,cores=2,threads=2 -uuid e03b486b-fdb7-4e30-8142-b55a5a7ce9fd -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=32,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=localtime,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=22,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=23,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device pcie-root-port,port=24,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x3 -device pcie-root-port,port=25,chassis=10,id=pci.10,bus=pcie.0,addr=0x3.0x1 -device pcie-root-port,port=26,chassis=11,id=pci.11,bus=pcie.0,addr=0x3.0x2 -device pcie-root-port,port=27,chassis=12,id=pci.12,bus=pcie.0,addr=0x3.0x3 -device pcie-root-port,port=28,chassis=13,id=pci.13,bus=pcie.0,addr=0x3.0x4 -device pcie-root-port,port=29,chassis=14,id=pci.14,bus=pcie.0,addr=0x3.0x5 -device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/Gaming.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","driver":"qcow2","file":"libvirt-2-storage","backing":null} -device ide-hd,bus=ide.0,drive=libvirt-2-format,id=sata0-0-0,bootindex=1 -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/windows.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device ide-cd,bus=ide.1,drive=libvirt-1-format,id=sata0-0-1 -netdev tap,fd=33,id=hostnet0 -device e1000e,netdev=hostnet0,id=net0,mac=52:54:00:5d:26:da,bus=pci.1,addr=0x0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev spicevmc,id=charchannel0,name=vdagent -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.spice.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -audiodev {"id":"audio1","driver":"spice"} -vnc 127.0.0.1:0,audiodev=audio1 -spice port=5901,addr=127.0.0.1,disable-ticketing=on,image-compression=off,seamless-migration=on -device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1 -device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0,audiodev=audio1 -chardev spicevmc,id=charredir0,name=usbredir -device usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev spicevmc,id=charredir1,name=usbredir -device usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device vfio-pci,host=0000:01:00.1,id=hostdev0,bus=pci.5,addr=0x0 -device vfio-pci,host=0000:01:00.0,id=hostdev1,bus=pci.6,addr=0x0 -device virtio-balloon-pci,id=balloon0,bus=pci.4,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on
 
@MarkusF

Very weird, but switching to ide from sata or virtio for windows boot disk and uninstalling and reinstalling virtio driver with memory baloon and virtgpu uninstalled or not selected works fine and no. crash

Very weird this sata to ide thing i guess is the main thing that prevents crash weidly

although i am using Virtio SCSI single

Do you know what can cause or whats causing this issue

Thanks
 
I am glad you could be resolve your issue.

I am sorry, I am not a Windows expert and it is hard to find out what the problem is without the source code ;)
Seem to be some windows driver issue.
 
@MarkusF I installed fresh Ubuntu 24 and KVM/Libvirt and copied the same Windows image to it with virtio and NVIDIA GPU passthough
And no crash, it works like charm and super stable!!!

This means some issue with PROXMOX + HOST CPU (AMD) + VIRTIO/SATA + GPU/PASSTHROUGH

where as this combination works

PROXMOX + QEMU/X86CPU (Virtual CPU) + VIRTIO/SATA + GPU/PASSTHROUGH

PROXMOX + HOST CPU (AMD) + IDE + GPU/PASSTHROUGH


Any idea why is this weird thing


I tried it on 3 AMD systems, 1 epyc, and zen 3 and zen 4 Ryzen and its easily reproducible

1. Simply install windows 10/11 + NVIDIA GPU passthrough then install a application for example GPU-Z or nested anything

2. Reboot

3. Once rebooted, open gpu-z and refresh 2-3 times

4. Wait till 11 mins of system boot, and exactly at 11mins the system will reboot with BSOD





Happens on old 6.5/5.13 all kernals and PVE 7.2 / 8.1 / 8.2
 
Last edited:
Hi,
@MarkusF I installed fresh Ubuntu 24 and KVM/Libvirt and copied the same Windows image to it with virtio and NVIDIA GPU passthough
And no crash, it works like charm and super stable!!!
what kernel and QEMU version did you use there?
This means some issue with PROXMOX + HOST CPU (AMD) + VIRTIO/SATA + GPU/PASSTHROUGH

where as this combination works

PROXMOX + QEMU/X86CPU (Virtual CPU) + VIRTIO/SATA + GPU/PASSTHROUGH

PROXMOX + HOST CPU (AMD) + IDE + GPU/PASSTHROUGH


Any idea why is this weird thing


I tried it on 3 AMD systems, 1 epyc, and zen 3 and zen 4 Ryzen and its easily reproducible

1. Simply install windows 10/11 + NVIDIA GPU passthrough then install a application for example GPU-Z or nested anything

2. Reboot

3. Once rebooted, open gpu-z and refresh 2-3 times

4. Wait till 11 mins of system boot, and exactly at 11mins the system will reboot with BSOD





Happens on old 6.5/5.13 all kernals and PVE 7.2 / 8.1 / 8.2
There is a package of QEMU 9.0 on the pvetest repository, you could try and see if that resolves the issue.
 
@fiona currently i am in

proxmox-ve: 8.1.0 (running kernel: 6.5.11-8-pve)
pve-manager: 8.1.4 (running version: 8.1.4/ec5affc9e41f1d79)
proxmox-kernel-helper: 8.1.0
proxmox-kernel-6.5: 6.5.11-8
proxmox-kernel-6.5.11-8-pve-signed: 6.5.11-8
ceph-fuse: 17.2.7-pve2
corosync: 3.1.7-pve3
criu: 3.17.1-2
glusterfs-client: 10.3-5
ifupdown2: 3.2.0-1+pmx8
ksm-control-daemon: 1.4-1
libjs-extjs: 7.0.0-4
libknet1: 1.28-pve1
libproxmox-acme-perl: 1.5.0
libproxmox-backup-qemu0: 1.4.1
libproxmox-rs-perl: 0.3.3
libpve-access-control: 8.0.7
libpve-apiclient-perl: 3.3.1
libpve-common-perl: 8.1.0
libpve-guest-common-perl: 5.0.6
libpve-http-server-perl: 5.0.5
libpve-network-perl: 0.9.5
libpve-rs-perl: 0.8.8
libpve-storage-perl: 8.0.5
libspice-server1: 0.15.1-1
lvm2: 2.03.16-2
lxc-pve: 5.0.2-4
lxcfs: 5.0.3-pve4
novnc-pve: 1.4.0-3
proxmox-backup-client: 3.1.4-1
proxmox-backup-file-restore: 3.1.4-1
proxmox-kernel-helper: 8.1.0
proxmox-mail-forward: 0.2.3
proxmox-mini-journalreader: 1.4.0
proxmox-offline-mirror-helper: 0.6.4
proxmox-widget-toolkit: 4.1.3
pve-cluster: 8.0.5
pve-container: 5.0.8
pve-docs: 8.1.3
pve-edk2-firmware: 4.2023.08-3
pve-firewall: 5.0.3
pve-firmware: 3.9-1
pve-ha-manager: 4.0.3
pve-i18n: 3.2.0
pve-qemu-kvm: 8.1.5-2
pve-xtermjs: 5.3.0-3
qemu-server: 8.0.10
smartmontools: 7.3-pve1
spiceterm: 3.3.0
swtpm: 0.8.0+pve1
vncterm: 1.8.0
zfsutils-linux: 2.2.2-pve1



let me try switching to test and see
 
@fiona currently i am in

proxmox-ve: 8.1.0 (running kernel: 6.5.11-8-pve)
pve-manager: 8.1.4 (running version: 8.1.4/ec5affc9e41f1d79)
proxmox-kernel-helper: 8.1.0
proxmox-kernel-6.5: 6.5.11-8
proxmox-kernel-6.5.11-8-pve-signed: 6.5.11-8
ceph-fuse: 17.2.7-pve2
corosync: 3.1.7-pve3
criu: 3.17.1-2
glusterfs-client: 10.3-5
ifupdown2: 3.2.0-1+pmx8
ksm-control-daemon: 1.4-1
libjs-extjs: 7.0.0-4
libknet1: 1.28-pve1
libproxmox-acme-perl: 1.5.0
libproxmox-backup-qemu0: 1.4.1
libproxmox-rs-perl: 0.3.3
libpve-access-control: 8.0.7
libpve-apiclient-perl: 3.3.1
libpve-common-perl: 8.1.0
libpve-guest-common-perl: 5.0.6
libpve-http-server-perl: 5.0.5
libpve-network-perl: 0.9.5
libpve-rs-perl: 0.8.8
libpve-storage-perl: 8.0.5
libspice-server1: 0.15.1-1
lvm2: 2.03.16-2
lxc-pve: 5.0.2-4
lxcfs: 5.0.3-pve4
novnc-pve: 1.4.0-3
proxmox-backup-client: 3.1.4-1
proxmox-backup-file-restore: 3.1.4-1
proxmox-kernel-helper: 8.1.0
proxmox-mail-forward: 0.2.3
proxmox-mini-journalreader: 1.4.0
proxmox-offline-mirror-helper: 0.6.4
proxmox-widget-toolkit: 4.1.3
pve-cluster: 8.0.5
pve-container: 5.0.8
pve-docs: 8.1.3
pve-edk2-firmware: 4.2023.08-3
pve-firewall: 5.0.3
pve-firmware: 3.9-1
pve-ha-manager: 4.0.3
pve-i18n: 3.2.0
pve-qemu-kvm: 8.1.5-2
pve-xtermjs: 5.3.0-3
qemu-server: 8.0.10
smartmontools: 7.3-pve1
spiceterm: 3.3.0
swtpm: 0.8.0+pve1
vncterm: 1.8.0
zfsutils-linux: 2.2.2-pve1
Sorry, I meant the versions when you were using Ubuntu 24/Libvirt, so we can compare.
 
@fiona

Tried pve-test

No its still the same



Also my ubuntu

Description: Ubuntu 24.04 LTS
Release: 24.04
Codename: noble
Kernal 6.8.0-31-generic
QEMU emulator version 8.2.2 (Debian 1:8.2.2+ds-0ubuntu1)
libvirtd (libvirt) 10.0.0



Here is my VM

<domain type='kvm'>
<name>Windows10Gaming</name>
<uuid>a17fcf3c-3f88-4f48-b145-c90d736de4e1</uuid>
<metadata xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0" xmlns:cockpit_machines="https://github.com/cockpit-project/cockpit-machi>
<libosinfo:libosinfo>
<libosinfo:os id="http://microsoft.com/win/10"/>
</libosinfo:libosinfo>
<cockpit_machines:data>
<cockpit_machines:has_install_phase>false</cockpit_machines:has_install_phase>
<cockpit_machines:install_source_type>file</cockpit_machines:install_source_type>
<cockpit_machines:os_variant>win10</cockpit_machines:os_variant>
</cockpit_machines:data>
</metadata>
<memory unit='KiB'>16777216</memory>
<currentMemory unit='KiB'>16777216</currentMemory>
<vcpu placement='static'>8</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='2'/>
<vcpupin vcpu='1' cpuset='3'/>
<vcpupin vcpu='2' cpuset='4'/>
<vcpupin vcpu='3' cpuset='5'/>
<vcpupin vcpu='4' cpuset='6'/>
<vcpupin vcpu='5' cpuset='7'/>
<vcpupin vcpu='6' cpuset='8'/>
<vcpupin vcpu='7' cpuset='9'/>
</cputune>
<os firmware='efi'>
<type arch='x86_64' machine='pc-q35-8.2'>hvm</type>
<firmware>
<feature enabled='yes' name='enrolled-keys'/>
<feature enabled='yes' name='secure-boot'/>
</firmware>
<loader readonly='yes' secure='yes' type='pflash'>/usr/share/OVMF/OVMF_CODE_4M.ms.fd</loader>
<nvram template='/usr/share/OVMF/OVMF_VARS_4M.ms.fd'>/var/lib/libvirt/qemu/nvram/Windows10Gaming_VARS.fd</nvram>
<boot dev='hd'/>
<smbios mode='host'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv mode='custom'>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<runtime state='on'/>
<synic state='on'/>
<stimer state='on'/>
<reset state='on'/>
<frequencies state='on'/>
</hyperv>
<kvm>
<hidden state='on'/>
</kvm>
<vmport state='off'/>
<smm state='on'/>
<vmcoreinfo state='on'/>
</features>
<cpu mode='host-passthrough' check='none' migratable='on'>
<topology sockets='1' dies='1' cores='4' threads='2'/>
<feature policy='require' name='topoext'/>
</cpu>
<clock offset='localtime'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
<timer name='hypervclock' present='yes'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='writeback' discard='unmap'/>
<source file='/var/lib/libvirt/images/Windows10Gaming.qcow2'/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x18'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x19'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x1a'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x1b'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x1c'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
</controller>
<controller type='pci' index='14' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='14' port='0x1d'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x5'/>
</controller>