LVM Archive filling up

Apr 25, 2025
1
0
1
Setup:
  • 3 Node VE (8.3.5) Cluster with Ceph and additional 3PAR Storage (Fibre Channel exposed to the Host)
Problem:
On one of the nodes, the `/etc/lvm/archive` contains 894035 files amounting to 3.5 GiB of space.

Code:
...
-rw------- 1 root root     1442 Apr  4 20:03 vg0_99990-2087856352.vg
-rw------- 1 root root     1442 Apr  4 20:03 vg0_99991-958218471.vg
-rw------- 1 root root     1839 Apr  4 20:03 vg0_99992-1653286176.vg
-rw------- 1 root root     1839 Apr  4 20:03 vg0_99993-33973345.vg
-rw------- 1 root root     1442 Apr  4 20:03 vg0_99994-217911204.vg
-rw------- 1 root root     1442 Apr  4 20:04 vg0_99995-367929124.vg
-rw------- 1 root root     1839 Apr  4 20:04 vg0_99996-395883521.vg
-rw------- 1 root root     1839 Apr  4 20:04 vg0_99997-377000313.vg
-rw------- 1 root root     1442 Apr  4 20:04 vg0_99998-813479538.vg
-rw------- 1 root root     1442 Apr  4 20:04 vg0_99999-972654829.vg
...

They appear to contain information about the LVM inside VMs:

Code:
# Generated by LVM2 version 2.03.16(2) (2022-05-18): Fri Apr  4 20:03:56 2025

contents = "Text Format Volume Group"
version = 1

description = "Created *before* executing '/sbin/vgs --separator : --noheadings --units b --unbuffered --nosuffix --options vg_name,vg_size,vg_free,lv_count'"

creation_host = "test-hy-2010"  # Linux test-hy-2010 6.8.12-8-pve #1 SMP PREEMPT_DYNAMIC PMX 6.8.12-8 (2025-01-24T12:32Z) x86_64
creation_time = 1743789836      # Fri Apr  4 20:03:56 2025

vg0 {
        id = "TSDRJx-NkNn-FUIo-24aW-Q35D-RQ7E-Xxof2d"
        seqno = 3
        format = "lvm2"                 # informational
        status = ["RESIZEABLE", "READ", "WRITE"]
        flags = []
        extent_size = 8192              # 4 Megabytes
        max_lv = 0
        max_pv = 0
        metadata_copies = 0

        physical_volumes {

                pv0 {
                        id = "QMpb3E-yqbN-vN4U-V1fM-330U-UU6W-RSgrZ0"
                        device = "/dev/mapper/3P05--intit--102-vm--106--disk--0p3"      # Hint only

                        status = ["ALLOCATABLE"]
                        flags = []
                        dev_size = 62908416     # 29.9971 Gigabytes
                        pe_start = 2048
                        pe_count = 7679 # 29.9961 Gigabytes
                }
        }

        logical_volumes {

                root {
                        id = "jdHkr6-azlC-7ryO-EY1D-wsWU-ThVw-HOKlJR"
                        status = ["READ", "WRITE", "VISIBLE"]
                        flags = []
                        creation_time = 1710256143      # 2024-03-12 16:09:03 +0100
                        creation_host = "ubuntu-server"
                        segment_count = 1

                        segment1 {
                                start_extent = 0
                                extent_count = 5120     # 20 Gigabytes

                                type = "striped"
                                stripe_count = 1        # linear

                                stripes = [
                                        "pv0", 0
                                ]
                        }
                }

                log {
                        id = "sg1LDH-7wbm-8Eph-NJXh-gTph-SdIf-CS4AVW"
                        status = ["READ", "WRITE", "VISIBLE"]
                        flags = []
                        creation_time = 1710256144      # 2024-03-12 16:09:04 +0100
                        creation_host = "ubuntu-server"
                        segment_count = 1

                        segment1 {
                                start_extent = 0
                                extent_count = 1280     # 5 Gigabytes

                                type = "striped"
                                stripe_count = 1        # linear

                                stripes = [
                                        "pv0", 5120
                                ]
                        }
                }
        }

}

lsblk for 3PAR Disk

Code:
sdg                                                                                                     8:96   0     4T  0 disk
└─3P05-intit-102                                                                                      252:10   0     4T  0 mpath
  ├─3P05--intit--102-vm--101--disk--0                                                                 252:13   0    32G  0 lvm
  ├─3P05--intit--102-vm--201--disk--0                                                                 252:14   0    16G  0 lvm
  ├─3P05--intit--102-vm--201--disk--1                                                                 252:15   0   160G  0 lvm
  ├─3P05--intit--102-vm--202--disk--0                                                                 252:16   0    25G  0 lvm
  ├─3P05--intit--102-vm--204--disk--0                                                                 252:17   0    35G  0 lvm
  ├─3P05--intit--102-vm--207--disk--0                                                                 252:18   0    20G  0 lvm
  ├─3P05--intit--102-vm--206--disk--0                                                                 252:20   0    20G  0 lvm
  ├─3P05--intit--102-vm--106--disk--0                                                                 252:21   0    32G  0 lvm
  ├─3P05--intit--102-vm--210--disk--0                                                                 252:22   0    30G  0 lvm
  ├─3P05--intit--102-vm--213--disk--0                                                                 252:23   0    16G  0 lvm
  ├─3P05--intit--102-vm--214--disk--0                                                                 252:24   0    25G  0 lvm
  ├─3P05--intit--102-vm--215--disk--0                                                                 252:25   0     8G  0 lvm
  ├─3P05--intit--102-vm--216--disk--0                                                                 252:26   0    20G  0 lvm
  ├─3P05--intit--102-vm--217--disk--0                                                                 252:27   0   512G  0 lvm
  ├─3P05--intit--102-vm--218--disk--0                                                                 252:28   0    25G  0 lvm
  ├─3P05--intit--102-vm--221--disk--0                                                                 252:29   0    50G  0 lvm
  ├─3P05--intit--102-vm--142--disk--0                                                                 252:30   0    32G  0 lvm
  └─3P05--intit--102-vm--145--disk--0                                                                 252:31   0   100G  0 lvm

Here are some statistics about the files inside `/etc/lvm/archive`.
First Column: Number of files inside `/etc/lvm/archive`

Code:
 178790                         device = "/dev/mapper/3P05--intit--102-vm--106--disk--0p3"      #
 178809                         device = "/dev/mapper/3P05--intit--102-vm--201--disk--0p2"      # 
 178806                         device = "/dev/mapper/3P05--intit--102-vm--203--disk--0p2"      # This storage does not exist anymore?
 178798                         device = "/dev/mapper/3P05--intit--102-vm--210--disk--0p2"      #

  1. LVM seems to be gathering information about the LVMs on the 3PAR but only for 4 VMs out of 20 VMs with 3PAR Storage
  2. I reduced the retain_days inside lvm.conf to 10 days but it does not seem to do any clean up
  3. Is it safe to delete data inside `/etc/lvm/archive/*` manually?
 
Last edited: