Proxmox's web interface has stopped working, how can I fix it?

[anton_eng]

First on one node, then the web interface started to fall off, then on the second, after which I tried to restart pveproxy, on the only node with a working web interface and on this node the service also stopped restarting.

Now there are hanging processes like:
32112 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
And it is not possible to delete these processes.

Spoiler: This is how the list of processes on one of the nodes looks like

PID TTY STAT TIME COMMAND
1 ? Ss 358:10 /lib/systemd/systemd --system --deserialize 19

340 ? Ss 68:44 /lib/systemd/systemd-journald
343 ? I< 0:39 [kworker/1:1H]
356 ? I< 0:00 [iscsi_eh]
364 ? I< 0:00 [rpciod]
365 ? I< 0:00 [xprtiod]
368 ? I< 0:00 [ib-comp-wq]
369 ? I< 0:00 [ib_mcast]
370 ? I< 0:00 [ib_nl_sa_wq]
374 ? I< 0:00 [rdma_cm]
382 ? S< 0:00 [spl_system_task]
383 ? S< 0:00 [spl_delay_taskq]
384 ? S< 0:00 [spl_dynamic_tas]
385 ? S< 0:00 [spl_kmem_cache]
411 ? I< 0:43 [kworker/2:1H]
443 ? I< 0:00 [ttm_swap]
448 ? SN 0:00 [kipmi0]
449 ? S< 0:00 [zvol]
450 ? S 0:00 [arc_prune]
451 ? S 45:09 [arc_reclaim]
452 ? S 0:00 [dbu_evict]
453 ? SN 29:45 [dbuf_evict]
583 ? SN 0:00 [z_vdev_file]
584 ? S 51:32 [l2arc_feed]
616 ? I< 0:00 [dm_bufio_cache]
621 ? I< 0:00 [kdmflush]
623 ? I< 0:00 [bioset]
625 ? I< 0:00 [kdmflush]
626 ? I< 0:00 [bioset]
639 ? I< 0:00 [kdmflush]
640 ? I< 0:00 [bioset]
643 ? I< 0:00 [kcopyd]
644 ? I< 0:00 [bioset]
645 ? I< 0:00 [dm-thin]
647 ? I< 0:00 [bioset]
648 ? I< 0:00 [kdmflush]
649 ? I< 0:00 [bioset]
653 ? SLsl 675:12 /sbin/dmeventd -f
866 ? Ss 3:11 /sbin/rpcbind -f -w
963 ? Ssl 0:00 /usr/bin/lxcfs /var/lib/lxcfs/
969 ? Ss 39:03 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
997 ? S 55:41 /bin/bash /usr/sbin/ksmtuned
1019 ? Ss 65:59 /usr/sbin/watchdog-mux
1020 ? Ss 22:47 /lib/systemd/systemd-logind
1029 ? Ss 2:34 /usr/sbin/smartd -n
1031 ? Ssl 13:21 /usr/sbin/rsyslogd -n
1346 ? I< 1:43 [kworker/4:1H]
1411 ? Ss 0:00 /usr/lib/x86_64-linux-gnu/lxc/lxc-monitord --daemon
1491 ? Ss 53:01 /sbin/iscsid
1492 ? S 1651 tty1 Ss+ 0:00 /sbin/agetty --noclear tty1 linux
1707 ? Ssl 105:46 /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 109:115
1724 ? Ssl 709:33 /usr/bin/rrdcached -B -b /var/lib/rrdcached/db/ -j /var/lib/rrdcached/journal/ -p /var/run/rrdcached.pid -l unix:/var/run/rrdcached.sock
1763 ? I< 0:41 [kworker/0:1H]
1790 ? Ss 8:54 /usr/lib/postfix/sbin/master -w
1799 ? S 3:01 qmgr -l -t unix -u
1814 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
1819 ? Ssl 2828:12 /usr/bin/pmxcfs
1832 ? RLsl 32287:23 /usr/sbin/corosync -f
1837 ? Ss 5:41 /usr/sbin/cron -f
1895 ? Ss 9021:18 pvestatd
1899 ? Ss 3327:18 pve-firewall
1929 ? Ss 34:49 pvedaemon
1950 ? I< 1:18 [kworker/5:1H]
1958 ? Ss 249:47 pve-ha-crm
2000 ? Ss 501:36 pve-ha-lrm
2005 ? Ss 34:02 spiceproxy
2206 ? I< 1:16 [kworker/7:1H]
2255 ? I< 1:10 [kworker/6:1H]
2723 ? I 0:00 [kworker/4:1]
2764 ? I 0:03 [kworker/2:1]
2829 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
3461 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
3484 ? I< 0:00 [kdmflush]
3489 ? I< 0:00 [bioset]
4335 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
8525 ? I 0:00 [kworker/5:0]
8584 ? I 0:00 [kworker/7:2]
8789 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
9925 ? Ss 0:00 sshd: drongo [priv]
9936 ? Ss 0:00 /lib/systemd/systemd --user
9937 ? S 0:00 (sd-pam)
9952 ? S 0:00 sshd: drongo@pts/1
9954 pts/1 Ss 0:00 -sh
9957 ? I 0:00 [kworker/3:1]
9959 pts/1 S 0:00 su -
9965 pts/1 S+ 0:00 -su
10912 ? I 0:00 [kworker/u17:2]
11198 ? I 0:00 [kworker/u16:1]
11292 ? I 0:00 [kworker/u17:0]
11559 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
11713 ? Sl 194145:19 /usr/bin/kvm -id 101 -name micsrv101 -chardev socket,id=qmp,path=/var/run/qemu-server/101.qmp,server,nowait -mon chardev=qmp,mode=control -pidfile /var/run/qemu-server/101.pid -daemonize -smbios type=1,u
11850 ? S 2034:32 [vhost-11713]
11855 ? S 284:13 [kvm-pit/11713]
12195 ? I 0:00 [kworker/u18:2]
13969 ? D 0:00 /usr/bin/perl -T /usr/bin/pveproxy status
14159 ? I 0:00 [kworker/u16:0]
14187 ? S 0:00 pickup -l -t unix -u -c
14202 ? I 0:00 [kworker/u18:1]
14426 ? I 0:00 [kworker/u17:1]
14495 ? I 0:00 [kworker/u16:2]
14512 ? I 0:00 [kworker/6:1]
14563 ? I 0:00 [kworker/0:0]
14602 ? I 0:00 [kworker/u18:0]
14667 ? I 0:00 [kworker/4:0]
14673 ? I 0:00 [kworker/5:1]
14688 ? Rs 0:00 sshd: root@pts/2
14695 ? Ss 0:00 sshd: root@notty
14705 pts/2 Ss 0:00 -bash
14731 ? Ss 0:00 /usr/lib/openssh/sftp-server
14734 ? I 0:00 [kworker/3:2]
14775 ? Ss 0:00 bash -c while [ -d /proc/$PPID ]; do sleep 1;head -v -n 8 /proc/meminfo; head -v -n 2 /proc/stat /proc/version /proc/uptime /proc/loadavg /proc/sys/fs/file-nr /proc/sys/kernel/hostname; tail -v -n 16 /proc/
14875 ? S 0:00 sleep 60
15077 ? S 0:00 sleep 1
15083 pts/2 R+ 0:00 ps ax
15202 ? R 1:03 [kworker/0:1]
15669 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pvesr run --mail 1
16040 ? Ss 334:16 /lib/systemd/systemd-udevd
17290 ? D 8:31 pvedaemon worker
17895 ? Ss 0:00 /usr/sbin/sshd -D
19645 ? D 16:53 pvedaemon worker
21765 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy restart
21872 ? Ssl 0:01 /usr/sbin/pvefw-logger
21928 ? S 0:00 spiceproxy worker
22807 ? I 0:00 [kworker/7:1]
22814 ? I 0:00 [kworker/5:2]
22843 ? I 0:00 [kworker/1:1]
22887 ? I 0:00 [kworker/6:2]
23929 ? I< 0:00 [kdmflush]
23934 ? I< 0:00 [bioset]
24793 ? I 0:00 [kworker/4:2]
24825 ? Ss 0:00 /lib/systemd/systemd --user
24826 ? S 0:00 (sd-pam)
26099 ? D 16:35 pvedaemon worker
27921 ? I< 0:00 [kdmflush]
27925 ? I< 0:00 [bioset]
30901 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy stop
31649 ? I 0:00 [kworker/3:0]
31658 ? I 0:00 [kworker/2:0]
31659 ? I 0:00 [kworker/1:2]
31691 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
32112 ? Ds 0:00 /usr/bin/perl -T /usr/bin/pveproxy start
32389 ? Ds 0:00 /usr/bin/perl /usr/bin/pveupdate