[SOLVED] local fs full: ghost files edition

aixz · Dec 9, 2023

Hello,

I'm having a local partition full problem

my 100Gb local fs is 95% full, but when I use ncdu it says I roughly use 30Gb
after checking with ncdu, I didn't find any abnormal artefact or tmp files, nothing to delete. To my understanding, 60Gb are ghost data that I can't find.

I looked at various posts about that and unfortunately found nothing that matches my usecase, I only found how to truncate log files ...

What could be wrong ? How to purge this ?

Code:

df -h
Filesystem            Size  Used Avail Use% Mounted on
udev                   16G     0   16G   0% /dev
tmpfs                 3.2G  3.1M  3.2G   1% /run
/dev/mapper/pve-root   98G   88G  5.2G  95% /
tmpfs                  16G   34M   16G   1% /dev/shm
tmpfs                 5.0M     0  5.0M   0% /run/lock
efivarfs              192K   42K  146K  23% /sys/firmware/efi/efivars
/dev/nvme0n1p2        511M  336K  511M   1% /boot/efi
/dev/nvme1n1p1        1.8T  1.3T  480G  73% /mnt/samsung
/dev/sde1             1.8T   36K  1.7T   1% /mnt/usb-backup2to
/dev/sdb1              19T   17T  304G  99% /mnt/20to
/dev/sdc1             932G  492G  440G  53% /mnt/backupusb
/dev/sda1             7.3T  6.9T   37G 100% /mnt/8to
/dev/sdd1             3.7T  3.0T  697G  82% /mnt/4tox
/dev/fuse             128M   32K  128M   1% /etc/pve
tmpfs                 3.2G     0  3.2G   0% /run/user/0

Code:

ncdu /

28.4 TiB [#########################] /mnt
.  26.8 GiB [                         ] /var
    4.7 GiB [                         ] /usr
  328.9 MiB [                         ] /boot
   33.5 MiB [                         ] /dev
   10.3 MiB [                         ] /etc
    3.1 MiB [                         ] /run
  604.0 KiB [                         ] /root
   44.0 KiB [                         ] /tmp
   40.0 KiB [                         ]  proxmox_toolbox.sh
   36.0 KiB [                         ] /home
e  16.0 KiB [                         ] /lost+found
e   4.0 KiB [                         ] /zfspool
e   4.0 KiB [                         ] /zdata
e   4.0 KiB [                         ] /srv
e   4.0 KiB [                         ] /opt
e   4.0 KiB [                         ] /media
    4.0 KiB [                         ]  syno.sh
.   0.0   B [                         ] /proc
    0.0   B [                         ] /sys
@   0.0   B [                         ]  libx32
@   0.0   B [                         ]  lib64
@   0.0   B [                         ]  lib32
@   0.0   B [                         ]  sbin
@   0.0   B [                         ]  lib
@   0.0   B [                         ]  bin

ncdu /mnt

   16.9 TiB [#########################] /20to
    6.8 TiB [##########               ] /8to
    3.0 TiB [####                     ] /4tox
    1.2 TiB [#                        ] /samsung
  491.8 GiB [                         ] /backupusb
   28.0 KiB [                         ] /usb-backup2to
    8.0 KiB [                         ] /pve
e   4.0 KiB [                         ] /vzsnap0
e   4.0 KiB [                         ] /syno-video
e   4.0 KiB [                         ] /syno-music
e   4.0 KiB [                         ] /syno-docs

ncdu /var/

.  20.7 GiB [#########################] /lib
    3.2 GiB [###                      ] /cache
    2.8 GiB [###                      ] /log
    3.3 MiB [                         ] /spool
    2.3 MiB [                         ] /backups
   32.0 KiB [                         ] /tmp
e   4.0 KiB [                         ] /opt
e   4.0 KiB [                         ] /mail
e   4.0 KiB [                         ] /local
@   0.0   B [                         ]  lock
@   0.0   B [                         ]  run

ncdu /var/lib

   20.6 GiB [#########################] /vz
   82.8 MiB [                         ] /apt
   35.5 MiB [                         ] /dpkg
    8.1 MiB [                         ] /smartmontools
    6.0 MiB [                         ] /rrdcached
    2.2 MiB [                         ] /samba
    1.7 MiB [                         ] /pve-cluster
    1.0 MiB [                         ] /lxc
  712.0 KiB [                         ] /usbutils
  652.0 KiB [                         ] /systemd
  128.0 KiB [                         ] /ucf
  112.0 KiB [                         ] /pve-manager
   84.0 KiB [                         ] /nfs
   32.0 KiB [                         ] /swtpm-localca
   28.0 KiB [                         ] /pam
   20.0 KiB [                         ] /libxml-sax-perl
   16.0 KiB [                         ] /ifupdown2
   16.0 KiB [                         ] /qemu-server
   12.0 KiB [                         ] /grub
   12.0 KiB [                         ] /dictionaries-common
    8.0 KiB [                         ] /vim
    8.0 KiB [                         ] /sudo
    8.0 KiB [                         ] /bluetooth
    8.0 KiB [                         ] /logrotate
    8.0 KiB [                         ] /sgml-base
    8.0 KiB [                         ] /chrony
    8.0 KiB [                         ] /postfix
    8.0 KiB [                         ] /dbus
    4.0 KiB [                         ] /python
e   4.0 KiB [                         ] /pve-firewall
e   4.0 KiB [                         ] /pve
e   4.0 KiB [                         ] /private
e   4.0 KiB [                         ] /misc
    4.0 KiB [                         ] /man-db
e   4.0 KiB [                         ] /git
e   4.0 KiB [                         ] /dhcp
e   4.0 KiB [                         ] /corosync
e   4.0 KiB [                         ] /ceph
    4.0 KiB [                         ]  shells.state
.   0.0   B [                         ] /lxcfs

ncdu /var/lib/vz/

   20.6 GiB [#########################] /template
e   4.0 KiB [                         ] /snippets
e   4.0 KiB [                         ] /private
e   4.0 KiB [                         ] /images
e   4.0 KiB [                         ] /dump

ncdu /var/lib/vz/template

   20.3 GiB [#########################] /iso
  363.6 MiB [                         ] /cache

ncdu /var/lib/vz/template/iso

    5.1 GiB [#########################]  Win11_22H2_French_x64.iso
    4.8 GiB [#######################  ]  ubuntu-23.10.1-desktop-amd64.iso
    3.6 GiB [#################        ]  tiny11_b2.iso
    3.2 GiB [###############          ]  debian-live-12.2.0-amd64-gnome.iso
    2.5 GiB [############             ]  tiny10_21H2_x64_2209.iso
  598.5 MiB [##                       ]  virtio-win-0.1.240.iso
  388.0 MiB [#                        ]  debian-11.6.0-amd64-netinst.iso

Code:

lsblk

NAME                         MAJ:MIN RM   SIZE RO TYPE MOUNTPOINTS
sda                            8:0    0   7.3T  0 disk
└─sda1                         8:1    0   7.3T  0 part /mnt/8to
sdb                            8:16   0  18.2T  0 disk
└─sdb1                         8:17   0  18.2T  0 part /mnt/20to
sdc                            8:32   0 931.5G  0 disk
└─sdc1                         8:33   0 931.5G  0 part /mnt/backupusb
sdd                            8:48   0   3.6T  0 disk
└─sdd1                         8:49   0   3.6T  0 part /mnt/4tox
sde                            8:64   0   1.8T  0 disk
└─sde1                         8:65   0   1.8T  0 part /mnt/usb-backup2to
nvme1n1                      259:0    0   1.8T  0 disk
└─nvme1n1p1                  259:1    0   1.8T  0 part /mnt/samsung
nvme0n1                      259:2    0 465.8G  0 disk
├─nvme0n1p1                  259:3    0  1007K  0 part
├─nvme0n1p2                  259:4    0   512M  0 part /boot/efi
└─nvme0n1p3                  259:5    0 465.3G  0 part
  ├─pve-swap                 252:0    0   7.6G  0 lvm  [SWAP]
  ├─pve-root                 252:1    0   100G  0 lvm  /
  ├─pve-data_tmeta           252:2    0   3.4G  0 lvm
  │ └─pve-data-tpool         252:4    0 334.8G  0 lvm
  │   ├─pve-data             252:5    0 334.8G  1 lvm
  │   ├─pve-vm--200--disk--0 252:6    0     8G  0 lvm
  │   ├─pve-vm--201--disk--0 252:7    0     4G  0 lvm
  │   ├─pve-vm--202--disk--0 252:8    0     6G  0 lvm
  │   ├─pve-vm--205--disk--0 252:9    0     4G  0 lvm
  │   ├─pve-vm--206--disk--0 252:10   0     8G  0 lvm
  │   ├─pve-vm--207--disk--0 252:11   0     3G  0 lvm
  │   ├─pve-vm--208--disk--0 252:12   0    10G  0 lvm
  │   ├─pve-vm--209--disk--0 252:13   0     2G  0 lvm
  │   ├─pve-vm--210--disk--0 252:14   0    40G  0 lvm
  │   ├─pve-vm--211--disk--0 252:15   0     8G  0 lvm
  │   ├─pve-vm--212--disk--0 252:16   0     2G  0 lvm
  │   ├─pve-vm--300--disk--0 252:17   0    60G  0 lvm
  │   ├─pve-vm--301--disk--0 252:18   0    60G  0 lvm
  │   ├─pve-vm--110--disk--0 252:19   0    32G  0 lvm
  │   ├─pve-vm--100--disk--0 252:20   0     4M  0 lvm
  │   ├─pve-vm--100--disk--1 252:21   0    32G  0 lvm
  │   ├─pve-vm--203--disk--0 252:22   0    12G  0 lvm
  │   ├─pve-vm--215--disk--0 252:23   0     2G  0 lvm
  │   ├─pve-vm--213--disk--0 252:24   0     8G  0 lvm
  │   ├─pve-vm--204--disk--0 252:25   0     8G  0 lvm
  │   └─pve-vm--214--disk--0 252:26   0     8G  0 lvm
  └─pve-data_tdata           252:3    0 334.8G  0 lvm
    └─pve-data-tpool         252:4    0 334.8G  0 lvm
      ├─pve-data             252:5    0 334.8G  1 lvm
      ├─pve-vm--200--disk--0 252:6    0     8G  0 lvm
      ├─pve-vm--201--disk--0 252:7    0     4G  0 lvm
      ├─pve-vm--202--disk--0 252:8    0     6G  0 lvm
      ├─pve-vm--205--disk--0 252:9    0     4G  0 lvm
      ├─pve-vm--206--disk--0 252:10   0     8G  0 lvm
      ├─pve-vm--207--disk--0 252:11   0     3G  0 lvm
      ├─pve-vm--208--disk--0 252:12   0    10G  0 lvm
      ├─pve-vm--209--disk--0 252:13   0     2G  0 lvm
      ├─pve-vm--210--disk--0 252:14   0    40G  0 lvm
      ├─pve-vm--211--disk--0 252:15   0     8G  0 lvm
      ├─pve-vm--212--disk--0 252:16   0     2G  0 lvm
      ├─pve-vm--300--disk--0 252:17   0    60G  0 lvm
      ├─pve-vm--301--disk--0 252:18   0    60G  0 lvm
      ├─pve-vm--110--disk--0 252:19   0    32G  0 lvm
      ├─pve-vm--100--disk--0 252:20   0     4M  0 lvm
      ├─pve-vm--100--disk--1 252:21   0    32G  0 lvm
      ├─pve-vm--203--disk--0 252:22   0    12G  0 lvm
      ├─pve-vm--215--disk--0 252:23   0     2G  0 lvm
      ├─pve-vm--213--disk--0 252:24   0     8G  0 lvm
      ├─pve-vm--204--disk--0 252:25   0     8G  0 lvm
      └─pve-vm--214--disk--0 252:26   0     8G  0 lvm

Code:

cat /etc/pve/storage.cfg

dir: local
        path /var/lib/vz
        content iso,vztmpl
        shared 0

lvmthin: local-lvm
        thinpool data
        vgname pve
        content rootdir,images

dir: backupusb
        path /mnt/backupusb
        content backup
        is_mountpoint 1
        mkdir 0
        prune-backups keep-all=1
        shared 1

nfs: usb-lapmox
        export /mnt/pve/usb
        path /mnt/pve/usb-lapmox
        server 192.168.1.90
        content backup
        mkdir 0
        prune-backups keep-all=1

dir: usb-backup2to
        path /mnt/usb-backup2to
        content backup
        is_mountpoint 1
        mkdir 0
        prune-backups keep-all=1
        shared 1

Code:

cat /etc/fstab

/dev/pve/root / ext4 errors=remount-ro 0 1
UUID=F4BA-F910 /boot/efi vfat defaults 0 1
/dev/pve/swap none swap sw 0 0
proc /proc proc defaults 0 0
#Samsung nvme 970EVO Plus
UUID=afebf45c-c8f1-4f0b-9f86-0b9ca12806f0 /mnt/samsung ext4 defaults 0 0
# Western 8TO
UUID=0700e121-85-c7-42c7-a004-cc7dcfd07981 /mnt/8to ext4 defaults,nofail 0 0
# 4TOX
UUID=3804E93504E8F6B2 /mnt/4tox ntfs defaults,nofail 0 0
# 20TO
UUID=93efe89c-ca80-40ef-8bb6-35960f6dc4d9 /mnt/20to ext4 defaults,nofail 0 0
# 1TO USB BACKUP
UUID=ECA807DDA807C4E4 /mnt/backupusb ntfs defaults,nofail 0 0
# 2TO USB BACKUP
UUID=99ad2380-264f-493a-b593-fc58dff6ed89 /mnt/usb-backup2to ext4 defaults,nofail 0 0

bbgeek17 · Dec 10, 2023

aixz said:
/dev/mapper/pve-root 98G 88G 5.2G 95% /

Your root partition is 98G
88G is used

aixz said:
26.8 GiB [ ] /var

aixz said:
4.7 GiB [ ] /usr

two significant parts account for 30G
You are still 58G short in your accounting.

Chances are that you wrote directly to one of the folders under /mnt when one of your external disks was NOT mounted.
Unmount all your disks, one at a time or all at the same time, then re-run your ndcu, or "du -h -x -d1 /mnt". I'd bet a donut that you will find your missing files.

Blockbridge : Ultra low latency all-NVME shared storage for Proxmox - https://www.blockbridge.com/proxmox

aixz · Dec 10, 2023

That's it.
i found duplicati's savefiles on one of the usb mounted drives. I wiped them and now I'm back to the normal 35Gb used on local fs.
I didn't thought of this.
Thank you very much !!

Search

Search

[SOLVED] local fs full: ghost files edition

aixz

Member

bbgeek17

Distinguished Member

aixz

Member