Hi,
I noticed that the WebGUI of my HomeLab showed all LXC containers offline. I checked and it looks like it is due to a kernel bug
The app servers running in those containers are still accessible.
I have tried to gather all details that might be relevant below
I noticed that the WebGUI of my HomeLab showed all LXC containers offline. I checked and it looks like it is due to a kernel bug
Code:
Oct 12 07:59:20 pve4 kernel: BUG: kernel NULL pointer dereference, address: 000000000000004c
Oct 12 07:59:21 pve4 kernel: #PF: supervisor read access in kernel mode
Oct 12 07:59:21 pve4 kernel: #PF: error_code(0x0000) - not-present pageThe app servers running in those containers are still accessible.
pct enter still workspct list does not workI have tried to gather all details that might be relevant below
Code:
root@pve4:~# ps -o pid,ppid,stat,wchan:32,etime,args -p 1911,223964
PID PPID STAT WCHAN ELAPSED COMMAND
1911 1 Ss poll_schedule_timeout.constprop. 2-21:49:55 pvestatd
223964 1911 D exit_mmap 24:55 [lxc-info]
root@pve4:~# cat /proc/223964/stack
[<0>] exit_mmap+0x163/0x400
[<0>] __mmput+0x43/0x160
[<0>] mmput+0x31/0x40
[<0>] do_exit+0x2b0/0xb20
[<0>] make_task_dead+0x93/0xa0
[<0>] rewind_stack_and_make_dead+0x16/0x20
root@pve4:~# tr '\0' ' ' </proc/223964/cmdline; echo
root@pve4:~# uname -r
7.0.14-23-pve
root@pve4:~# journalctl -k -b --since "2026-10-12 07:55" --no-pager
Oct 12 07:59:20 pve4 kernel: BUG: kernel NULL pointer dereference, address: 000000000000004c
Oct 12 07:59:21 pve4 kernel: #PF: supervisor read access in kernel mode
Oct 12 07:59:21 pve4 kernel: #PF: error_code(0x0000) - not-present page
Oct 12 07:59:21 pve4 kernel: PGD 0 P4D 0
Oct 12 07:59:21 pve4 kernel: Oops: Oops: 0000 [#1] SMP NOPTI
Oct 12 07:59:21 pve4 kernel: CPU: 5 UID: 0 PID: 223964 Comm: lxc-info Tainted: P O 7.0.14-23-pve #1 PREEMPT(lazy)
Oct 12 07:59:21 pve4 kernel: Tainted: [P]=PROPRIETARY_MODULE, [O]=OOT_MODULE
Oct 12 07:59:21 pve4 kernel: Hardware name: LENOVO 82FN/LNVNB161216, BIOS FFCN21WW(V1.09) 05/30/2022
Oct 12 07:59:21 pve4 kernel: RIP: 0010:charge_memcg+0x38/0x90
Oct 12 07:59:21 pve4 kernel: Code: 53 48 8b 07 48 89 f3 89 d6 ba 01 00 00 00 a8 40 74 03 8b 57 78 48 3b 1d d6 2f 1c 02 74 0c 48 89 df e8 cc f6 ff ff 85 c0 75 4a <f6> 43 4c 01 74 21 49 89 5c 24 38 48 89 de 4c 89 e7 e8 32 7a ff ff
Oct 12 07:59:21 pve4 kernel: RSP: 0018:ffffd2890f6d7a00 EFLAGS: 00010246
Oct 12 07:59:21 pve4 kernel: RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
Oct 12 07:59:21 pve4 kernel: RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
Oct 12 07:59:21 pve4 kernel: RBP: ffffd2890f6d7a10 R08: 0000000000000000 R09: 0000000000000000
Oct 12 07:59:21 pve4 kernel: R10: 0000000000000000 R11: 0000000000000000 R12: fffffcca84d6ddc0
Oct 12 07:59:21 pve4 kernel: R13: 0000000000000cc0 R14: ffff8bdbc8ddcd00 R15: ffff8bdb6ef901a8
Oct 12 07:59:21 pve4 kernel: FS: 0000790d568878c0(0000) GS:ffff8bde7158a000(0000) knlGS:0000000000000000
Oct 12 07:59:21 pve4 kernel: CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
Oct 12 07:59:21 pve4 kernel: CR2: 000000000000004c CR3: 000000036a1c7000 CR4: 0000000000350ef0
Oct 12 07:59:21 pve4 kernel: Call Trace:
Oct 12 07:59:21 pve4 kernel: <TASK>
Oct 12 07:59:21 pve4 kernel: __mem_cgroup_charge+0x2d/0xa0
Oct 12 07:59:21 pve4 kernel: do_fault+0xa4/0x580
Oct 12 07:59:21 pve4 kernel: __handle_mm_fault+0x9c4/0x1020
Oct 12 07:59:21 pve4 kernel: handle_mm_fault+0x136/0x3a0
Oct 12 07:59:21 pve4 kernel: __get_user_pages+0x7ef/0x1310
Oct 12 07:59:21 pve4 kernel: ? mt_find+0x20b/0x510
Oct 12 07:59:21 pve4 kernel: populate_vma_page_range+0x77/0xd0
Oct 12 07:59:21 pve4 kernel: __mm_populate+0xb5/0x190
Oct 12 07:59:21 pve4 kernel: vm_mmap_pgoff+0x182/0x210
Oct 12 07:59:21 pve4 kernel: ksys_mmap_pgoff+0x1bb/0x270
Oct 12 07:59:21 pve4 kernel: ? vfs_read+0x180/0x390
Oct 12 07:59:21 pve4 kernel: __x64_sys_mmap+0x33/0x70
Oct 12 07:59:21 pve4 kernel: x64_sys_call+0x1fc4/0x2390
Oct 12 07:59:21 pve4 kernel: do_syscall_64+0x10b/0x14e0
Oct 12 07:59:21 pve4 kernel: ? ksys_read+0x6f/0xf0
Oct 12 07:59:21 pve4 kernel: ? __x64_sys_read+0x19/0x30
Oct 12 07:59:21 pve4 kernel: ? x64_sys_call+0x1ff1/0x2390
Oct 12 07:59:21 pve4 kernel: ? do_syscall_64+0x148/0x14e0
Oct 12 07:59:21 pve4 kernel: ? exc_page_fault+0x92/0x1c0
Oct 12 07:59:21 pve4 kernel: entry_SYSCALL_64_after_hwframe+0x76/0x7e
Oct 12 07:59:21 pve4 kernel: RIP: 0033:0x790d56b9eb62
Oct 12 07:59:21 pve4 kernel: Code: 00 00 00 0f 1f 44 00 00 41 f7 c1 ff 0f 00 00 75 27 55 89 cd 53 48 89 fb 48 85 ff 74 3b 41 89 ea 48 89 df b8 09 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 76 5b 5d c3 0f 1f 00 48 8b 05 79 82 0d 00 64
Oct 12 07:59:21 pve4 kernel: RSP: 002b:00007fff998c7628 EFLAGS: 00000246 ORIG_RAX: 0000000000000009
Oct 12 07:59:21 pve4 kernel: RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 0000790d56b9eb62
Oct 12 07:59:21 pve4 kernel: RDX: 0000000000000003 RSI: 00000000000000c6 RDI: 0000000000000000
Oct 12 07:59:21 pve4 kernel: RBP: 0000000000008002 R08: 000000000000000c R09: 0000000000000000
Oct 12 07:59:21 pve4 kernel: R10: 0000000000008002 R11: 0000000000000246 R12: 00007fff998c7f90
Oct 12 07:59:21 pve4 kernel: R13: 000000000000000c R14: 0000790d56e53628 R15: 00000000000000c5
Oct 12 07:59:21 pve4 kernel: </TASK>
Oct 12 07:59:21 pve4 kernel: Modules linked in: tcp_diag inet_diag nf_conntrack_netlink xt_MASQUERADE xfrm_user xfrm_algo nft_chain_nat nft_compat overlay veth ebtable_filter ebtables ip6table_raw ip6t_REJECT nf_reject_ipv6 ip6table_filter ip6_tables xt_mac xt_connmark ipt_REJECT nf_reject_ipv4 xt_set xt_physdev xt_addrtype xt_multiport xt_conntrack xt_comment xt_tcpudp xt_mark iptable_filter ip_set_hash_net ip_set sctp nf_tables softdog vrf binfmt_misc vxlan ip6_udp_tunnel udp_tunnel iptable_raw xt_CT iptable_nat xt_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 bonding tls nfnetlink_log nfsd auth_rpcgss nfs_acl lockd grace snd_acp_legacy_mach sunrpc snd_acp_mach snd_soc_nau8821 snd_soc_dmic snd_acp3x_pdm_dma snd_acp3x_rn snd_sof_amd_acp70 snd_sof_amd_acp63 snd_sof_amd_vangogh amdgpu snd_sof_amd_rembrandt snd_sof_amd_renoir snd_sof_amd_acp snd_sof_pci snd_sof_xtensa_dsp snd_sof snd_sof_utils snd_pci_ps snd_soc_acpi_amd_match snd_amd_sdw_acpi soundwire_amd soundwire_generic_allocation soundwire_bus snd_hda_codec_alc269
Oct 12 07:59:21 pve4 kernel: snd_hda_codec_realtek_lib snd_soc_sdca snd_hda_scodec_component snd_hda_codec_generic snd_hda_codec_atihdmi amdxcp snd_soc_core snd_hda_codec_hdmi drm_panel_backlight_quirks snd_compress gpu_sched snd_hda_intel ac97_bus uvcvideo drm_buddy snd_pcm_dmaengine amd_atl intel_rapl_msr snd_hda_codec videobuf2_vmalloc drm_ttm_helper snd_rpl_pci_acp6x btusb intel_rapl_common uvc ttm snd_acp_pci snd_hda_core videobuf2_memops btmtk snd_amd_acpi_mach drm_exec iwlmvm snd_intel_dspcfg videobuf2_v4l2 btrtl snd_acp_legacy_common drm_suballoc_helper snd_intel_sdw_acpi edac_mce_amd videobuf2_common snd_pci_acp6x snd_hwdep btbcm drm_display_helper mac80211 snd_pci_acp5x snd_pcm btintel videodev kvm_amd snd_rn_pci_acp3x snd_timer cec lenovo_ymc libarc4 hid_sensor_accel_3d rc_core snd_acp_config mc bluetooth snd snd_soc_acpi hid_sensor_trigger i2c_algo_bit ccp kvm snd_pci_acp3x soundcore hid_sensor_iio_common irqbypass iwlwifi ghash_clmulni_intel bmc150_accel_spi aesni_intel bmc150_accel_i2c rapl pcspkr
Oct 12 07:59:21 pve4 kernel: lenovo_wmi_hotkey_utilities wmi_bmof ideapad_laptop bmc150_accel_core cfg80211 ee1004 industrialio_triggered_buffer k10temp sparse_keymap platform_profile kfifo_buf industrialio joydev soc_button_array amd_pmc input_leds sch_fq_codel mac_hid msr vhost_net vhost vhost_iotlb tap efi_pstore nfnetlink zstd zram 842_decompress 842_compress lz4hc_compress lz4_compress dmi_sysfs ip_tables x_tables autofs4 zfs(PO) spl(O) btrfs libblake2b xor raid6_pq r8153_ecm cdc_ether usbnet uas usb_storage r8152 mii wacom hid_sensor_custom usbhid hid_multitouch hid_sensor_hub hid_generic nvme ucsi_acpi nvme_core xhci_pci typec_ucsi i2c_hid_acpi typec i2c_piix4 nvme_keyring nvme_auth video serio_raw i2c_hid xhci_hcd i2c_smbus thunderbolt hkdf wmi hid
Oct 12 07:59:21 pve4 kernel: CR2: 000000000000004c
Oct 12 07:59:21 pve4 kernel: ---[ end trace 0000000000000000 ]---
Oct 12 07:59:21 pve4 kernel: RIP: 0010:charge_memcg+0x38/0x90
Oct 12 07:59:21 pve4 kernel: Code: 53 48 8b 07 48 89 f3 89 d6 ba 01 00 00 00 a8 40 74 03 8b 57 78 48 3b 1d d6 2f 1c 02 74 0c 48 89 df e8 cc f6 ff ff 85 c0 75 4a <f6> 43 4c 01 74 21 49 89 5c 24 38 48 89 de 4c 89 e7 e8 32 7a ff ff
Oct 12 07:59:21 pve4 kernel: RSP: 0018:ffffd2890f6d7a00 EFLAGS: 00010246
Oct 12 07:59:21 pve4 kernel: RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
Oct 12 07:59:21 pve4 kernel: RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
Oct 12 07:59:21 pve4 kernel: RBP: ffffd2890f6d7a10 R08: 0000000000000000 R09: 0000000000000000
Oct 12 07:59:21 pve4 kernel: R10: 0000000000000000 R11: 0000000000000000 R12: fffffcca84d6ddc0
Oct 12 07:59:21 pve4 kernel: R13: 0000000000000cc0 R14: ffff8bdbc8ddcd00 R15: ffff8bdb6ef901a8
Oct 12 07:59:21 pve4 kernel: FS: 0000790d568878c0(0000) GS:ffff8bde7158a000(0000) knlGS:0000000000000000
Oct 12 07:59:21 pve4 kernel: CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
Oct 12 07:59:21 pve4 kernel: CR2: 000000000000004c CR3: 000000036a1c7000 CR4: 0000000000350ef0
Oct 12 07:59:21 pve4 kernel: note: lxc-info[223964] exited with irqs disabled
root@pve4:~# cat /proc/cmdline
BOOT_IMAGE=/vmlinuz-7.0.14-23-pve root=ZFS=/ROOT/pve-1 ro root=ZFS=rpool/ROOT/pve-1 boot=zfs quiet
root@pve4:~# grep -R . /sys/fs/cgroup/memory.events 2>/dev/null
root@pve4:~# ps aux | grep lxc-info
root 223964 0.1 0.0 0 0 ? D 07:59 0:02 [lxc-info]
root 237987 0.0 0.0 6528 1884 pts/3 S+ 08:25 0:00 grep lxc-info
root@pve4:~#
root@pve4:~# pveversion -v
proxmox-ve: 9.2.0 (running kernel: 7.0.14-23-pve)
pve-manager: 9.2.21 (running version: 9.2.21/4f6e0ac86f9e8c7f)
proxmox-kernel-helper: 9.2.0
proxmox-kernel-7.0: 7.0.14-23
proxmox-kernel-7.0.14-23-pve-signed: 7.0.14-23
proxmox-kernel-7.0.14-22-pve-signed: 7.0.14-22
amd64-microcode: 3.20251202.1~bpo13+1
ceph-fuse: 19.2.6-pve4
corosync: 3.1.10-pve3
criu: 4.1.1-1
dnsmasq: 2.91-1+deb13u2
frr-pythontools: 10.6.1-1+pve3
ifupdown2: 3.3.0-1+pmx12
ksm-control-daemon: 1.5-1
libjs-extjs: 7.0.0-7
libproxmox-acme-perl: 1.7.2
libproxmox-backup-qemu0: 2.0.3
libproxmox-rs-perl: 0.4.1
libpve-access-control: 9.1.2
libpve-apiclient-perl: 3.4.3
libpve-cluster-api-perl: 9.1.6
libpve-cluster-perl: 9.1.6
libpve-common-perl: 9.2.3
libpve-guest-common-perl: 6.0.5
libpve-http-server-perl: 6.0.5
libpve-network-perl: 1.6.7
libpve-notify-perl: 9.1.6
libpve-rs-perl: 0.15.3
libpve-storage-perl: 9.1.12
libspice-server1: 0.15.2-1+b1
lvm2: 2.03.31-2+pmx1
lxc-pve: 7.0.0-2
lxcfs: 7.0.0-pve1
novnc-pve: 1.7.0-2
proxmox-backup-client: 4.2.8-1
proxmox-backup-file-restore: 4.2.8-1
proxmox-backup-restore-image: 1.0.0
proxmox-enterprise-support-keyring: 1.1
proxmox-firewall: 1.2.3
proxmox-kernel-helper: 9.2.0
proxmox-mail-forward: 1.0.3
proxmox-mini-journalreader: 1.7
proxmox-offline-mirror-helper: 0.7.4
proxmox-widget-toolkit: 5.2.10
pve-cluster: 9.1.6
pve-container: 6.1.14
pve-docs: 9.2.13
pve-edk2-firmware: 4.2026.08-1
pve-esxi-import-tools: 1.0.1
pve-firewall: 6.0.6
pve-firmware: 3.18-7
pve-ha-manager: 5.2.5
pve-i18n: 3.10.0
pve-qemu-kvm: 11.0.3-4
pve-xtermjs: 6.0.0-2
qemu-server: 9.2.10
smartmontools: 7.5-pve2
spiceterm: 3.4.2
swtpm: 0.8.0+pve3
vncterm: 1.9.2
zfsutils-linux: 2.4.4-pve1