VM state goes to "io-error" but disk space should be fine

P

PartitionPrankster

New Member
Proxmox Subscriber
Jul 11, 2025
6
0
1
Hey,

on a shared storage with hundreds of VMs, I have one VM that seem to constantly switch to the "io-error" state.

My setup:

I have a non-thin LVM, that is connected via FibreChannel to the host.
The physical volume is 8TB big. Only ~3TB is assigned to VMs, so I don't think it's an issue of a full host disk.
Also, there are hundreds of other VMs on the same storage - they don't experience any issues.

What I checked so far:

echo '{"execute": "qmp_capabilities"}{"execute": "query-block", "arguments": {}}' | socat - /var/run/qemu-server/107.qmp | jq .
{
"QMP": {
"version": {
"qemu": {
"micro": 2,
"minor": 0,
"major": 10
},
"package": "pve-qemu-kvm_10.0.2-4"
},
"capabilities": []
}
}
{
"return": {}
}
{
"return": [
{
"io-status": "ok",
"device": "",
"locked": true,
"removable": true,
"inserted": {
"iops_rd": 0,
"detect_zeroes": "off",
"active": true,
"image": {
"backing-image": {
"virtual-size": 8388608,
"filename": "/dev/storage1-preview-1/vm-107-cloudinit",
"format": "raw",
"actual-size": 0,
"dirty-flag": false
},
"virtual-size": 8388608,
"filename": "json:{\"throttle-group\": \"throttle-drive-ide2\", \"driver\": \"throttle\", \"file\": {\"driver\": \"raw\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-cloudinit\"}}}",
"format": "throttle",
"actual-size": 0,
"dirty-flag": false
},
"iops_wr": 0,
"ro": true,
"children": [
{
"node-name": "fad7a45e2bca0b0efb685abc8d9b2ef",
"child": "file"
}
],
"node-name": "drive-ide2",
"backing_file_depth": 1,
"drv": "throttle",
"iops": 0,
"bps_wr": 0,
"write_threshold": 0,
"encrypted": false,
"bps": 0,
"bps_rd": 0,
"cache": {
"no-flush": false,
"direct": false,
"writeback": true
},
"file": "json:{\"throttle-group\": \"throttle-drive-ide2\", \"driver\": \"throttle\", \"file\": {\"driver\": \"raw\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-cloudinit\"}}}"
},
"qdev": "ide2",
"tray_open": false,
"type": "unknown"
},
{
"io-status": "nospace",
"device": "",
"locked": false,
"removable": false,
"inserted": {
"iops_rd": 0,
"detect_zeroes": "on",
"active": true,
"image": {
"backing-image": {
"virtual-size": 10737418240,
"filename": "/dev/storage1-preview-1/vm-107-disk-0.qcow2",
"cluster-size": 65536,
"format": "qcow2",
"actual-size": 0,
"format-specific": {
"type": "qcow2",
"data": {
"compat": "1.1",
"compression-type": "zlib",
"lazy-refcounts": false,
"refcount-bits": 16,
"corrupt": false,
"extended-l2": false
}
},
"dirty-flag": false
},
"virtual-size": 10737418240,
"filename": "json:{\"throttle-group\": \"throttle-drive-scsi0\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-disk-0.qcow2\"}}}",
"cluster-size": 65536,
"format": "throttle",
"actual-size": 0,
"dirty-flag": false
},
"iops_wr": 0,
"ro": false,
"children": [
{
"node-name": "f8b934fb1f6846ccfef3594d3efebdb",
"child": "file"
}
],
"node-name": "drive-scsi0",
"backing_file_depth": 1,
"drv": "throttle",
"iops": 0,
"bps_wr": 0,
"write_threshold": 0,
"encrypted": false,
"bps": 0,
"bps_rd": 0,
"cache": {
"no-flush": false,
"direct": false,
"writeback": true
},
"file": "json:{\"throttle-group\": \"throttle-drive-scsi0\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-disk-0.qcow2\"}}}"
},
"qdev": "scsi0",
"type": "unknown"
},
{
"io-status": "ok",
"device": "",
"locked": false,
"removable": false,
"inserted": {
"iops_rd": 0,
"detect_zeroes": "on",
"active": true,
"image": {
"backing-image": {
"virtual-size": 4294967296,
"filename": "/dev/storage1-preview-1/vm-107-disk-2.qcow2",
"cluster-size": 65536,
"format": "qcow2",
"actual-size": 0,
"format-specific": {
"type": "qcow2",
"data": {
"compat": "1.1",
"compression-type": "zlib",
"lazy-refcounts": false,
"refcount-bits": 16,
"corrupt": false,
"extended-l2": false
}
},
"dirty-flag": false
},
"virtual-size": 4294967296,
"filename": "json:{\"throttle-group\": \"throttle-drive-scsi2\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-disk-2.qcow2\"}}}",
"cluster-size": 65536,
"format": "throttle",
"actual-size": 0,
"dirty-flag": false
},
"iops_wr": 0,
"ro": false,
"children": [
{
"node-name": "f842cc8aee5c625dfe52be2caf346a0",
"child": "file"
}
],
"node-name": "drive-scsi2",
"backing_file_depth": 1,
"drv": "throttle",
"iops": 0,
"bps_wr": 0,
"write_threshold": 0,
"encrypted": false,
"bps": 0,
"bps_rd": 0,
"cache": {
"no-flush": false,
"direct": false,
"writeback": true
},
"file": "json:{\"throttle-group\": \"throttle-drive-scsi2\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-disk-2.qcow2\"}}}"
},
"qdev": "scsi2",
"type": "unknown"
},
{
"io-status": "ok",
"device": "",
"locked": false,
"removable": false,
"inserted": {
"iops_rd": 0,
"detect_zeroes": "on",
"active": true,
"image": {
"backing-image": {
"virtual-size": 54760833024,
"filename": "/dev/storage1-preview-1/vm-107-disk-1.qcow2",
"cluster-size": 65536,
"format": "qcow2",
"actual-size": 0,
"format-specific": {
"type": "qcow2",
"data": {
"compat": "1.1",
"compression-type": "zlib",
"lazy-refcounts": false,
"refcount-bits": 16,
"corrupt": false,
"extended-l2": false
}
},
"dirty-flag": false
},
"virtual-size": 54760833024,
"filename": "json:{\"throttle-group\": \"throttle-drive-scsi3\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-disk-1.qcow2\"}}}",
"cluster-size": 65536,
"format": "throttle",
"actual-size": 0,
"dirty-flag": false
},
"iops_wr": 0,
"ro": false,
"children": [
{
"node-name": "fa1c283d5f1584b679fce04d2b3145c",
"child": "file"
}
],
"node-name": "drive-scsi3",
"backing_file_depth": 1,
"drv": "throttle",
"iops": 0,
"bps_wr": 0,
"write_threshold": 0,
"encrypted": false,
"bps": 0,
"bps_rd": 0,
"cache": {
"no-flush": false,
"direct": false,
"writeback": true
},
"file": "json:{\"throttle-group\": \"throttle-drive-scsi3\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"host_device\", \"filename\": \"/dev/storage1-preview-1/vm-107-disk-1.qcow2\"}}}"
},
"qdev": "scsi3",
"type": "unknown"
}
]
}
Click to expand...

lvdisplay /dev/storage1-preview-1/vm-107-disk-0.qcow2
root@n-1:~# lvdisplay /dev/storage1-preview-1/vm-107-disk-0.qcow2
--- Logical volume ---
LV Path /dev/storage1-preview-1/vm-107-disk-0.qcow2
LV Name vm-107-disk-0.qcow2
VG Name storage1-preview-1
LV UUID 3tYkgm-C0Af-2YKD-6Dfo-Wtph-3BZ0-dPJ7f3
LV Write Access read/write
LV Creation host, time n-1, 2025-07-23 14:50:14 +0200
LV Status available
# open 1
LV Size 10.00 GiB
Current LE 2561
Segments 1
Allocation inherit
Read ahead sectors auto
- currently set to 16384
Block device 252:7
Click to expand...

cat /proc/partitions (inside the VM)
major minor #blocks name

8 0 10485760 sda
8 1 1024 sda1
8 2 10482688 sda2
8 16 4194304 sdb
8 32 53477376 sdc
8 33 52427759 sdc1
11 0 8192 sr0
Click to expand...

root@n-1:~# pvdisplay /dev/mapper/storage1-preview-1
--- Physical volume ---
PV Name /dev/mapper/pure3_b-preview-1
VG Name pure3_b-preview-1
PV Size 8.00 TiB / not usable 4.00 MiB
Allocatable yes
PE Size 4.00 MiB
Total PE 2097151
Free PE 1318762
Allocated PE 778389
PV UUID jJCHNo-NHUf-tjTO-0nFd-6xj2-MYDJ-Yk80A1
Click to expand...

root@n-1:~# pvdisplay --columns --all
PV VG Fmt Attr PSize PFree
/dev/mapper/pure3_b-preview-1 pure3_b-preview-1 lvm2 a-- <8.00t 5.03t
/dev/sda2 --- 0 0
/dev/sda3 pve lvm2 a-- <446.07g 16.00g
Click to expand...


root@n-1:~# vgdisplay storage1-preview-1
--- Volume group ---
VG Name storage1-preview-1
System ID
Format lvm2
Metadata Areas 1
Metadata Sequence No 804
VG Access read/write
VG Status resizable
MAX LV 0
Cur LV 511
Open LV 100
Max PV 0
Cur PV 1
Act PV 1
VG Size <8.00 TiB
PE Size 4.00 MiB
Total PE 2097151
Alloc PE / Size 778389 / <2.97 TiB
Free PE / Size 1318762 / 5.03 TiB
VG UUID ZqF5d2-M2Sr-DxGT-shls-1Lfg-qYyk-znWthJ
Click to expand...

My VMs config file:
root@n-1:~# cat /etc/pve/qemu-server/107.conf
agent: enabled=1
autostart: 1
balloon: 0
boot: order=scsi0
cicustom: user=pve-cloudinit:snippets/testvm.yaml
cores: 1
cpu: x86-64-v2-AES
hotplug: network,disk,cpu,memory,usb,cloudinit
ide2: storage1-preview-1:vm-107-cloudinit,media=cdrom
ipconfig0: gw=10.10.10.1,ip=10.10.10.10/20
memory: 9216
meta: creation-qemu=10.0.2,ctime=1753274457
migrate_downtime: 0.05
name: vmname.com
nameserver: 10.0.0.2
net0: virtio=BC:24:11:EA:04:50,bridge=testnet
numa: 1
ostype: l26
scsi0: storage1-preview-1:vm-107-disk-0.qcow2,aio=threads,iothread=1,size=10G
scsi2: storage1-preview-1:vm-107-disk-2.qcow2,aio=threads,iothread=1,size=4G
scsi3: storage1-preview-1:vm-107-disk-1.qcow2,aio=threads,iothread=1,size=51G
scsihw: virtio-scsi-single
searchdomain: example.com
smbios1: uuid=091c13c1-f47f-474d-a38c-62418d60a710
sockets: 4
tags: ops;ubuntu_24.04_lts
vmgenid: 430b01f0-5a90-4701-8bf2-240bae377af0
Click to expand...


So at least for me, it looks like there would be plenty of free space left on the host.
To me, the VM disk size and the LV size seem to match.

Is there anything I am missing here?

Thanks a lot in advance
 
You must log in or register to reply here.
Top Bottom