FYI: There's a newer version of the
pve-qemu-kvm
package (
6.0.0-4
) available through the
pvetest repository for Proxmox VE 7, it includes a proposed fix for a race found in the QMP stack of QEMU that would explain that issue happening occassionally under heavy load.
I updated to 6.0.0-4. It just broke a centOS VM.
INFO: resuming VM again
ERROR: VM 9327 qmp command 'cont' failed - got timeout
INFO: aborting backup job
INFO: resuming VM again
ERROR: Backup of VM 9327 failed - VM 9327 qmp command 'cont' failed - got timeout
The machine was not under a large load and it was the first backup of the set!
Detailed trace below!
VM_PID=541196
root@v1:~# gdb attach $VM_PID -ex='thread apply all bt' -ex='quit'
GNU gdb (Debian 10.1-1.7) 10.1.90.20210103-git
Copyright (C) 2021 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <
http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<
https://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<
http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
attach: No such file or directory.
Attaching to process 541196
[New LWP 541197]
[New LWP 541260]
[New LWP 541261]
[New LWP 541262]
[New LWP 541263]
[New LWP 541266]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007fb3508694f6 in __ppoll (fds=0x561d0fca0390, nfds=77, timeout=<optimized out>,
timeout@entry=0x7ffc4a71c370, sigmask=sigmask@entry=0x0) at ../sysdeps/unix/sysv/linux/ppoll.c:44
44 ../sysdeps/unix/sysv/linux/ppoll.c: No such file or directory.
Thread 7 (Thread 0x7fb33c9ff700 (LWP 541266) "kvm"):
#0 futex_wait_cancelable (private=0, expected=0, futex_word=0x561d10376748) at ../sysdeps/nptl/futex-internal.h:186
#1 __pthread_cond_wait_common (abstime=0x0, clockid=0, mutex=0x561d10376758, cond=0x561d10376720) at pthread_cond_wait.c:508
#2 __pthread_cond_wait (cond=cond@entry=0x561d10376720, mutex=mutex@entry=0x561d10376758) at pthread_cond_wait.c:638
#3 0x0000561d0d95bdfb in qemu_cond_wait_impl (cond=0x561d10376720, mutex=0x561d10376758, file=0x561d0da3f6f2 "../ui/vnc-jobs.c", line=248) at ../util/qemu-thread-posix.c:174
#4 0x0000561d0d55bbc3 in vnc_worker_thread_loop (queue=0x561d10376720) at ../ui/vnc-jobs.c:248
#5 0x0000561d0d55c888 in vnc_worker_thread (arg=arg@entry=0x561d10376720) at ../ui/vnc-jobs.c:361
#6 0x0000561d0d95b6b9 in qemu_thread_start (args=0x7fb33c9fa3f0) at ../util/qemu-thread-posix.c:521
#7 0x00007fb350943ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
#8 0x00007fb350873def in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Thread 6 (Thread 0x7fb33effd700 (LWP 541263) "kvm"):
#0 0x00007fb35086acc7 in ioctl () at ../sysdeps/unix/syscall-template.S:120
#1 0x0000561d0d7e9ac7 in kvm_vcpu_ioctl (cpu=cpu@entry=0x561d0f21ffc0, type=type@entry=44672) at ../accel/kvm/kvm-all.c:2630
#2 0x0000561d0d7e9c31 in kvm_cpu_exec (cpu=cpu@entry=0x561d0f21ffc0) at ../accel/kvm/kvm-all.c:2467
#3 0x0000561d0d787dad in kvm_vcpu_thread_fn (arg=arg@entry=0x561d0f21ffc0) at ../accel/kvm/kvm-accel-ops.c:49
#4 0x0000561d0d95b6b9 in qemu_thread_start (args=0x7fb33eff83f0) at ../util/qemu-thread-posix.c:521
#5 0x00007fb350943ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
#6 0x00007fb350873def in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Thread 5 (Thread 0x7fb33f7fe700 (LWP 541262) "kvm"):
--Type <RET> for more, q to quit, c to continue without paging--
#0 0x00007fb35086acc7 in ioctl () at ../sysdeps/unix/syscall-template.S:120
#1 0x0000561d0d7e9ac7 in kvm_vcpu_ioctl (cpu=cpu@entry=0x561d0f2124f0, type=type@entry=44672) at ../accel/kvm/kvm-all.c:2630
#2 0x0000561d0d7e9c31 in kvm_cpu_exec (cpu=cpu@entry=0x561d0f2124f0) at ../accel/kvm/kvm-all.c:2467
#3 0x0000561d0d787dad in kvm_vcpu_thread_fn (arg=arg@entry=0x561d0f2124f0) at ../accel/kvm/kvm-accel-ops.c:49
#4 0x0000561d0d95b6b9 in qemu_thread_start (args=0x7fb33f7f93f0) at ../util/qemu-thread-posix.c:521
#5 0x00007fb350943ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
#6 0x00007fb350873def in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Thread 4 (Thread 0x7fb33ffff700 (LWP 541261) "kvm"):
#0 0x00007fb35086acc7 in ioctl () at ../sysdeps/unix/syscall-template.S:120
#1 0x0000561d0d7e9ac7 in kvm_vcpu_ioctl (cpu=cpu@entry=0x561d0f203c80, type=type@entry=44672) at ../accel/kvm/kvm-all.c:2630
#2 0x0000561d0d7e9c31 in kvm_cpu_exec (cpu=cpu@entry=0x561d0f203c80) at ../accel/kvm/kvm-all.c:2467
#3 0x0000561d0d787dad in kvm_vcpu_thread_fn (arg=arg@entry=0x561d0f203c80) at ../accel/kvm/kvm-accel-ops.c:49
#4 0x0000561d0d95b6b9 in qemu_thread_start (args=0x7fb33fffa3f0) at ../util/qemu-thread-posix.c:521
#5 0x00007fb350943ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
#6 0x00007fb350873def in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Thread 3 (Thread 0x7fb345a47700 (LWP 541260) "kvm"):
#0 0x00007fb35086acc7 in ioctl () at ../sysdeps/unix/syscall-template.S:120
#1 0x0000561d0d7e9ac7 in kvm_vcpu_ioctl (cpu=cpu@entry=0x561d0f1cde30, type=type@entry=44672) at ../accel/kvm/kvm-all.c:2630
#2 0x0000561d0d7e9c31 in kvm_cpu_exec (cpu=cpu@entry=0x561d0f1cde30) at ../accel/kvm/kvm-all.c:2467
#3 0x0000561d0d787dad in kvm_vcpu_thread_fn (arg=arg@entry=0x561d0f1cde30) at ../accel/kvm/kvm-accel-ops.c:49
#4 0x0000561d0d95b6b9 in qemu_thread_start (args=0x7fb345a423f0) at ../util/qemu-thread-posix.c:521
#5 0x00007fb350943ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
#6 0x00007fb350873def in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Thread 2 (Thread 0x7fb346349700 (LWP 541197) "kvm"):
#0 syscall () at ../sysdeps/unix/sysv/linux/x86_64/syscall.S:38
#1 0x0000561d0d95c64a in qemu_futex_wait (val=<optimized out>, f=<optimized out>) at /build/pve-qemu/pve-qemu-kvm-6.0.0/include/qemu/futex.h:29
#2 qemu_event_wait (ev=ev@entry=0x561d0deed148 <rcu_call_ready_event>) at ../util/qemu-thread-posix.c:460
#3 0x0000561d0d98680a in call_rcu_thread (opaque=opaque@entry=0x0) at ../util/rcu.c:258
#4 0x0000561d0d95b6b9 in qemu_thread_start (args=0x7fb3463443f0) at ../util/qemu-thread-posix.c:521
#5 0x00007fb350943ea7 in start_thread (arg=<optimized out>) at pthread_create.c:477
#6 0x00007fb350873def in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Thread 1 (Thread 0x7fb3464b31c0 (LWP 541196) "kvm"):
#0 0x00007fb3508694f6 in __ppoll (fds=0x561d0fca0390, nfds=77, timeout=<optimized out>, timeout@entry=0x7ffc4a71c370, sigmask=sigmask@entry=0x0) at ../sysdeps/unix/sysv/linux/ppoll.c:44
#1 0x0000561d0d95d521 in ppoll (__ss=0x0, __timeout=0x7ffc4a71c370, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/x86_64-linux-gnu/bits/poll2.h:77
#2 qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=timeout@entry=1999362413) at ../util/qemu-timer.c:348
#3 0x0000561d0d9697e5 in os_host_main_loop_wait (timeout=1999362413) at ../util/main-loop.c:249
#4 main_loop_wait (nonblocking=nonblocking@entry=0) at ../util/main-loop.c:530
#5 0x0000561d0d7b5b91 in qemu_main_loop () at ../softmmu/runstate.c:725
#6 0x0000561d0d476c0e in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at ../softmmu/main.c:50
A debugging session is active.
Inferior 1 [process 541196] will be detached.
Quit anyway? (y or n) y
Detaching from program: /usr/bin/qemu-system-x86_64, process 541196
[Inferior 1 (process 541196) detached]