So I ran into a problem with running proxmox VM on my homelab.
I've created ZFS with 3 of my 500gb HDD drives. Everything went as planned except I stumbled upon system freezes, crashes, root-pve beeing filled up. (root-pve is installed as LVM)
When I was creating Ubuntu mirror using apt-mirror my server crashed. Upon rebooting I've discovered that It's once again "filled up" root partition.
I've had experienced this issue since I'd configured ZFS using built-in GUI. When inspecting I can't find any files (or directories) that are on "/" bigger than 5G (using this command "du -hsx /* | sort -rh | head -n 40"), but "df -h" returns 100% usage of "/" ("/dev/mapper/pve-root 49G 49G 0 100% /").
At first I assumed that this problem was swap related, but I deleted it from LVM and all of containers. Sometimes reboot does helps, sometimes it doesn't.
My system is up-to date. ZFS drives aren't damaged, nor my boot SSD. I've deleted some unimportant log files - such as /var/log/bandwidth but the drive usage is still at 100%. No containers' drives are on pve-root, and they're filled in few percents.
DF -H
root@pve:~# df -h
Filesystem Size Used Avail Use% Mounted on
udev 16G 0 16G 0% /dev
tmpfs 3.2G 98M 3.1G 4% /run
/dev/mapper/pve-root 49G 49G 0 100% /
tmpfs 16G 0 16G 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
ZFS 158G 256K 158G 1% /ZFS
ZFS/subvol-109-disk-0 305G 147G 158G 49% /ZFS/subvol-109-disk-0
ZFS/subvol-105-disk-0 162G 3.6G 158G 3% /ZFS/subvol-105-disk-0
ZFS/subvol-108-disk-0 30G 2.6G 28G 9% /ZFS/subvol-108-disk-0
ZFS/subvol-110-disk-0 40G 4.6G 36G 12% /ZFS/subvol-110-disk-0
ZFS/subvol-107-disk-0 650G 540G 111G 84% /ZFS/subvol-107-disk-0
ZFS/subvol-107-disk-1 8.0G 128K 8.0G 1% /ZFS/subvol-107-disk-1
tmpfs 3.2G 0 3.2G 0% /run/user/0
DU SOMETHING
root@pve:~# du -hsx /* | sort -rh | head -n 40
du: cannot access '/proc/4381/task/4381/fd/4': No such file or directory
du: cannot access '/proc/4381/task/4381/fdinfo/4': No such file or directory
du: cannot access '/proc/4381/fd/3': No such file or directory
du: cannot access '/proc/4381/fdinfo/3': No such file or directory
3.3G /usr
1.6G /var
391M /boot
146M /etc
106M /run
132K /root
56K /mnt
16K /lost+found
11K /ZFS
4.0K /tmp
4.0K /srv
4.0K /opt
4.0K /media
4.0K /home
4.0K /HDD
0 /sys
0 /sbin
0 /proc
0 /libx32
0 /lib64
0 /lib32
0 /lib
0 /dev
0 /bin
LSBLK
root@pve:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 465.8G 0 disk
├─sda1 8:1 0 465.8G 0 part
└─sda9 8:9 0 8M 0 part
sdb 8:16 0 465.8G 0 disk
├─sdb1 8:17 0 465.8G 0 part
└─sdb9 8:25 0 8M 0 part
sdc 8:32 0 111.8G 0 disk
├─sdc1 8:33 0 512M 0 part
└─sdc2 8:34 0 111.3G 0 part
sdd 8:48 0 465.8G 0 disk
└─sdd1 8:49 0 465.8G 0 part
sde 8:64 0 465.8G 0 disk
├─sde1 8:65 0 465.8G 0 part
└─sde9 8:73 0 8M 0 part
sdf 8:80 0 111.8G 0 disk
├─sdf1 8:81 0 1007K 0 part
├─sdf2 8:82 0 512M 0 part
└─sdf3 8:83 0 111.3G 0 part
├─pve-root 253:0 0 49.6G 0 lvm /
├─pve-data_tmeta 253:1 0 1G 0 lvm
│ └─pve-data-tpool 253:3 0 59.7G 0 lvm
│ ├─pve-data 253:4 0 59.7G 1 lvm
│ ├─pve-vm--100--disk--0 253:5 0 30G 0 lvm
│ ├─pve-vm--200--disk--4 253:6 0 4M 0 lvm
│ ├─pve-vm--106--disk--0 253:7 0 8G 0 lvm
│ └─pve-vm--300--disk--0 253:8 0 32G 0 lvm
└─pve-data_tdata 253:2 0 59.7G 0 lvm
└─pve-data-tpool 253:3 0 59.7G 0 lvm
├─pve-data 253:4 0 59.7G 1 lvm
├─pve-vm--100--disk--0 253:5 0 30G 0 lvm
├─pve-vm--200--disk--4 253:6 0 4M 0 lvm
├─pve-vm--106--disk--0 253:7 0 8G 0 lvm
└─pve-vm--300--disk--0 253:8 0 32G 0 lvm
zd0 230:0 0 32G 0 disk
├─zd0p1 230:1 0 31G 0 part
├─zd0p2 230:2 0 1K 0 part
└─zd0p5 230:5 0 975M 0 part
NOTE TO LSBLK: pve vm disk aren't full, so they aren' exceeding full drive capacity in any way
I've created ZFS with 3 of my 500gb HDD drives. Everything went as planned except I stumbled upon system freezes, crashes, root-pve beeing filled up. (root-pve is installed as LVM)
When I was creating Ubuntu mirror using apt-mirror my server crashed. Upon rebooting I've discovered that It's once again "filled up" root partition.
I've had experienced this issue since I'd configured ZFS using built-in GUI. When inspecting I can't find any files (or directories) that are on "/" bigger than 5G (using this command "du -hsx /* | sort -rh | head -n 40"), but "df -h" returns 100% usage of "/" ("/dev/mapper/pve-root 49G 49G 0 100% /").
At first I assumed that this problem was swap related, but I deleted it from LVM and all of containers. Sometimes reboot does helps, sometimes it doesn't.
My system is up-to date. ZFS drives aren't damaged, nor my boot SSD. I've deleted some unimportant log files - such as /var/log/bandwidth but the drive usage is still at 100%. No containers' drives are on pve-root, and they're filled in few percents.
DF -H
root@pve:~# df -h
Filesystem Size Used Avail Use% Mounted on
udev 16G 0 16G 0% /dev
tmpfs 3.2G 98M 3.1G 4% /run
/dev/mapper/pve-root 49G 49G 0 100% /
tmpfs 16G 0 16G 0% /dev/shm
tmpfs 5.0M 0 5.0M 0% /run/lock
ZFS 158G 256K 158G 1% /ZFS
ZFS/subvol-109-disk-0 305G 147G 158G 49% /ZFS/subvol-109-disk-0
ZFS/subvol-105-disk-0 162G 3.6G 158G 3% /ZFS/subvol-105-disk-0
ZFS/subvol-108-disk-0 30G 2.6G 28G 9% /ZFS/subvol-108-disk-0
ZFS/subvol-110-disk-0 40G 4.6G 36G 12% /ZFS/subvol-110-disk-0
ZFS/subvol-107-disk-0 650G 540G 111G 84% /ZFS/subvol-107-disk-0
ZFS/subvol-107-disk-1 8.0G 128K 8.0G 1% /ZFS/subvol-107-disk-1
tmpfs 3.2G 0 3.2G 0% /run/user/0
DU SOMETHING
root@pve:~# du -hsx /* | sort -rh | head -n 40
du: cannot access '/proc/4381/task/4381/fd/4': No such file or directory
du: cannot access '/proc/4381/task/4381/fdinfo/4': No such file or directory
du: cannot access '/proc/4381/fd/3': No such file or directory
du: cannot access '/proc/4381/fdinfo/3': No such file or directory
3.3G /usr
1.6G /var
391M /boot
146M /etc
106M /run
132K /root
56K /mnt
16K /lost+found
11K /ZFS
4.0K /tmp
4.0K /srv
4.0K /opt
4.0K /media
4.0K /home
4.0K /HDD
0 /sys
0 /sbin
0 /proc
0 /libx32
0 /lib64
0 /lib32
0 /lib
0 /dev
0 /bin
LSBLK
root@pve:~# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 465.8G 0 disk
├─sda1 8:1 0 465.8G 0 part
└─sda9 8:9 0 8M 0 part
sdb 8:16 0 465.8G 0 disk
├─sdb1 8:17 0 465.8G 0 part
└─sdb9 8:25 0 8M 0 part
sdc 8:32 0 111.8G 0 disk
├─sdc1 8:33 0 512M 0 part
└─sdc2 8:34 0 111.3G 0 part
sdd 8:48 0 465.8G 0 disk
└─sdd1 8:49 0 465.8G 0 part
sde 8:64 0 465.8G 0 disk
├─sde1 8:65 0 465.8G 0 part
└─sde9 8:73 0 8M 0 part
sdf 8:80 0 111.8G 0 disk
├─sdf1 8:81 0 1007K 0 part
├─sdf2 8:82 0 512M 0 part
└─sdf3 8:83 0 111.3G 0 part
├─pve-root 253:0 0 49.6G 0 lvm /
├─pve-data_tmeta 253:1 0 1G 0 lvm
│ └─pve-data-tpool 253:3 0 59.7G 0 lvm
│ ├─pve-data 253:4 0 59.7G 1 lvm
│ ├─pve-vm--100--disk--0 253:5 0 30G 0 lvm
│ ├─pve-vm--200--disk--4 253:6 0 4M 0 lvm
│ ├─pve-vm--106--disk--0 253:7 0 8G 0 lvm
│ └─pve-vm--300--disk--0 253:8 0 32G 0 lvm
└─pve-data_tdata 253:2 0 59.7G 0 lvm
└─pve-data-tpool 253:3 0 59.7G 0 lvm
├─pve-data 253:4 0 59.7G 1 lvm
├─pve-vm--100--disk--0 253:5 0 30G 0 lvm
├─pve-vm--200--disk--4 253:6 0 4M 0 lvm
├─pve-vm--106--disk--0 253:7 0 8G 0 lvm
└─pve-vm--300--disk--0 253:8 0 32G 0 lvm
zd0 230:0 0 32G 0 disk
├─zd0p1 230:1 0 31G 0 part
├─zd0p2 230:2 0 1K 0 part
└─zd0p5 230:5 0 975M 0 part
NOTE TO LSBLK: pve vm disk aren't full, so they aren' exceeding full drive capacity in any way