Very slow disk speed inside virtual machines

[Damo-HG]

Over the last week or two the servers speed and server load have gone up and up. CPU usage is very high even though there really isn't much running on the vms. Disk speeds seem to be very very slow inside virtual machines.

Block Size | 4k            (IOPS) | 64k           (IOPS)
  ------   | ---            ----  | ----           ----
Read       | 2.53 MB/s      (633) | 40.66 MB/s     (635)
Write      | 2.54 MB/s      (636) | 40.97 MB/s     (640)
Total      | 5.08 MB/s     (1.2k) | 81.63 MB/s    (1.2k)
           |                      |
Block Size | 512k          (IOPS) | 1m            (IOPS)
  ------   | ---            ----  | ----           ----
Read       | 198.54 MB/s    (387) | 193.51 MB/s    (188)
Write      | 209.09 MB/s    (408) | 206.40 MB/s    (201)
Total      | 407.64 MB/s    (795) | 399.91 MB/s    (389)

I've attempted to run pveperf however it's been stuck on the BUFFERED READS: part for over 30 minutes now. I will update if/when it's complete.

root@hostname:~# pveperf
CPU BOGOMIPS:      220380.00
REGEX/SECOND:      33386
HD SIZE:           39.25 GB (/dev/md2)

root@hostname:~# cat /proc/mounts
sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
udev /dev devtmpfs rw,nosuid,relatime,size=131880156k,nr_inodes=32970039,mode=755 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=26380700k,mode=755 0 0
/dev/md2 / ext4 rw,relatime 0 0
securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0
tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0
tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0
cgroup2 /sys/fs/cgroup/unified cgroup2 rw,nosuid,nodev,noexec,relatime 0 0
cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,name=systemd 0 0
pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0
efivarfs /sys/firmware/efi/efivars efivarfs rw,nosuid,nodev,noexec,relatime 0 0
none /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0
cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0
cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
cgroup /sys/fs/cgroup/hugetlb cgroup rw,nosuid,nodev,noexec,relatime,hugetlb 0 0
cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0
cgroup /sys/fs/cgroup/pids cgroup rw,nosuid,nodev,noexec,relatime,pids 0 0
cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0
cgroup /sys/fs/cgroup/rdma cgroup rw,nosuid,nodev,noexec,relatime,rdma 0 0
cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0
systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=30,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=45055 0 0
hugetlbfs /dev/hugepages hugetlbfs rw,relatime,pagesize=2M 0 0
sunrpc /run/rpc_pipefs rpc_pipefs rw,relatime 0 0
mqueue /dev/mqueue mqueue rw,relatime 0 0
debugfs /sys/kernel/debug debugfs rw,relatime 0 0
/dev/nvme0n1p1 /boot/efi vfat rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro 0 0
/dev/mapper/vg-data /var/lib/vz ext4 rw,relatime 0 0
fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0
configfs /sys/kernel/config configfs rw,relatime 0 0
lxcfs /var/lib/lxcfs fuse.lxcfs rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other 0 0
googlestroage:/hostname /mnt/pve/storage nfs4 rw,relatime,vers=4.2,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,clientaddr=51.89.xxx.xxx,local_lock=none,addr=54.38.xxx.xxx 0 0
tmpfs /run/user/0 tmpfs rw,nosuid,nodev,relatime,size=26380696k,mode=700 0 0
/dev/fuse /etc/pve fuse rw,nosuid,nodev,relatime,user_id=0,group_id=0,default_permissions,allow_other 0 0
//178.32.xxx.xxx/shared /mnt/pve/cifftes cifs rw,relatime,vers=3.0,cache=strict,username=proxmox,uid=0,noforceuid,gid=0,noforcegid,addr=178.32.xxx.xxx,file_mode=0755,dir_mode=0755,soft,nounix,serverino,mapposix,rsize=4194304,wsize=4194304,bsize=1048576,echo_interval=60,actimeo=1 0 0
//178.32.xxx.xxx/shared /mnt/pve/tete cifs rw,relatime,vers=3.0,cache=strict,username=proxmox,uid=0,noforceuid,gid=0,noforcegid,addr=178.32.xxx.xxx,file_mode=0755,dir_mode=0755,soft,nounix,serverino,mapposix,rsize=4194304,wsize=4194304,bsize=1048576,echo_interval=60,actimeo=1 0 0
//178.32.xxx.xxx/shared /mnt/pve/ciff cifs rw,relatime,vers=3.0,cache=strict,username=proxmox,uid=0,noforceuid,gid=0,noforcegid,addr=178.32.xxx.xxx,file_mode=0755,dir_mode=0755,soft,nounix,serverino,mapposix,rsize=4194304,wsize=4194304,bsize=1048576,echo_interval=60,actimeo=1 0 0
//178.32.xxx.xxx/shared /mnt/pve/uyuy cifs rw,relatime,vers=3.0,cache=strict,username=proxmox2,uid=0,noforceuid,gid=0,noforcegid,addr=178.32.xxx.xxx,file_mode=0755,dir_mode=0755,soft,nounix,serverino,mapposix,rsize=4194304,wsize=4194304,bsize=1048576,echo_interval=60,actimeo=1 0 0
//178.32.xxx.xxx/shared /mnt/pve/storage2 cifs rw,relatime,vers=3.0,cache=strict,username=proxmox2,uid=0,noforceuid,gid=0,noforcegid,addr=178.32.xxx.xxx,file_mode=0755,dir_mode=0755,soft,nounix,serverino,mapposix,rsize=4194304,wsize=4194304,bsize=1048576,echo_interval=60,actimeo=1 0 0
//178.32.xxx.xxx/shared /mnt/pve/storagewwq cifs rw,relatime,vers=3.0,cache=strict,username=proxmox2,uid=0,noforceuid,gid=0,noforcegid,addr=178.32.xxx.xxx,file_mode=0755,dir_mode=0755,soft,nounix,serverino,mapposix,rsize=4194304,wsize=4194304,bsize=1048576,echo_interval=60,actimeo=1 0 0
46.105.xxx.xxx:/hostname /mnt/pve/storage nfs rw,relatime,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=46.105.xxx.xxx,mountvers=3,mountport=43374,mountproto=udp,local_lock=none,addr=46.105.xxx.xxx 0 0
storage:/mnt/unionfs/backups/proxmox/hostname /var/lib/vz/backup fuse.rclone rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other 0 0

/etc/fstab

UUID="715bba1a-2748-482f-ae65-4d8e6f7571bf" / ext4 defaults 0 0
UUID="408d2a36-0812-42c1-9905-06a73f22d83d" /var/lib/vz ext4 defaults 0 0
UUID="6acb77eb-ef88-4428-b9eb-7955ed492e8d" swap swap defaults 0 0
UUID="b4af45c5-ae2e-4fee-b2f6-cda4244b03ae" swap swap defaults 0 0
LABEL=EFI_SYSPART /boot/efi vfat defaults 0 0

I've read adding barrier=0 would fix this? Should I add it to / ?

 

[wolfgang]

Hi,

I've read adding barrier=0 would fix this? Should I add it to / ?

This will not fix the problem. I would inspect what process consumes the disk io.
atop is a good tool for inspection performance issues.

 

[apoc]

CPU usage is very high even though there really isn't much running on the vms

What is the reason for this?
IO also needs CPU to process. So I would start looking there.