NODE 1
root@pve01:~# systemctl restart pve-cluster.service
root@pve01:~# systemctl status pve-cluster.service
- pve-cluster.service - The Proxmox VE cluster filesystem
Loaded: loaded (/lib/systemd/system/pve-cluster.service; enabled; vendor preset: enabled)
Active:
active (running) since Mon 2019-08-12 09:44:34 EDT; 13s ago
Process: 2145714 ExecStart=/usr/bin/pmxcfs (code=exited, status=0/SUCCESS)
Process: 2145727 ExecStartPost=/usr/bin/pvecm updatecerts --silent (code=exited, status=0/SUCCESS)
Main PID: 2145719 (pmxcfs)
Tasks: 6 (limit: 7372)
Memory: 19.2M
CGroup: /system.slice/pve-cluster.service └─2145719 /usr/bin/pmxcfs
Aug 12 09:44:33 pve01 pmxcfs[2145719]:
[status] crit: can't initialize service Aug 12 09:44:34 pve01 systemd[1]: Started The Proxmox VE cluster filesystem. Aug 12 09:44:39 pve01 pmxcfs[2145719]:
[quorum] crit: quorum_initialize failed: 2 Aug 12 09:44:39 pve01 pmxcfs[2145719]:
[confdb] crit: cmap_initialize failed: 2 Aug 12 09:44:39 pve01 pmxcfs[2145719]:
[dcdb] crit: cpg_initialize failed: 2 Aug 12 09:44:39 pve01 pmxcfs[2145719]:
[status] crit: cpg_initialize failed: 2 Aug 12 09:44:45 pve01 pmxcfs[2145719]:
[quorum] crit: quorum_initialize failed: 2 Aug 12 09:44:45 pve01 pmxcfs[2145719]:
[confdb] crit: cmap_initialize failed: 2 Aug 12 09:44:45 pve01 pmxcfs[2145719]:
[dcdb] crit: cpg_initialize failed: 2 Aug 12 09:44:45 pve01 pmxcfs[2145719]:
[status] crit: cpg_initialize failed: 2
-----------------------------------
NODE 2
root@pve02:~# systemctl restart pve-cluster.service Job for pve-cluster.service failed because the control process exited with error code.
See "systemctl status pve-cluster.service" and "journalctl -xe" for details.
root@pve02:~# systemctl status pve-cluster.service
● pve-cluster.service - The Proxmox VE cluster filesystem
Loaded: loaded (/lib/systemd/system/pve-cluster.service; enabled; vendor preset: enabled)
Active:
failed (Result: exit-code) since Mon 2019-08-12 09:52:53 EDT; 33s ago
Process: 217828 ExecStart=/usr/bin/pmxcfs
(code=exited, status=255/EXCEPTION)
Aug 12 09:52:53 pve02 systemd[1]: pve-cluster.service: Service RestartSec=100ms expired, scheduling restart.
Aug 12 09:52:53 pve02 systemd[1]: pve-cluster.service: Scheduled restart job, restart counter is at 5.
Aug 12 09:52:53 pve02 systemd[1]: Stopped The Proxmox VE cluster filesystem.
Aug 12 09:52:53 pve02 systemd[1]: pve-cluster.service: Start request repeated too quickly.
Aug 12 09:52:53 pve02 systemd[1]: pve-cluster.service: Failed with result 'exit-code'. Aug 12 09:52:53 pve02 systemd[1]:
Failed to start The Proxmox VE cluster filesystem.
root@pve02:~# journalctl -xe
-- Subject: A start job for unit pvesr.service has begun execution -- Defined-By: systemd
--
-- A start job for unit pvesr.service has begun execution.
--
-- The job identifier is 613367.
Aug 12 09:56:00 pve02 pvesr[221621]: ipcc_send_rec[1] failed: Connection refused
Aug 12 09:56:00 pve02 pvesr[221621]: ipcc_send_rec[2] failed: Connection refused
Aug 12 09:56:00 pve02 pvesr[221621]: ipcc_send_rec[3] failed: Connection refused
Aug 12 09:56:00 pve02 pvesr[221621]: Unable to load access control list: Connection refused
Aug 12 09:56:00 pve02 systemd[1]: pvesr.service: Main process exited, code=exited, status=111/n/a -- Subject: Unit process exited
-- Defined-By: systemd
--
-- An ExecStart= process belonging to unit pvesr.service has exited.
--
-- The process' exit code is 'exited' and its exit status is 111.
Aug 12 09:56:00 pve02 systemd[1]: pvesr.service: Failed with result 'exit-code'. -- Subject: Unit failed
-- Defined-By: systemd
--
-- The unit pvesr.service has entered the 'failed' state with result 'exit-code'.
Aug 12 09:56:00 pve02 systemd[1]:
Failed to start Proxmox VE replication runner. -- Subject: A start job for unit pvesr.service has failed
-- Defined-By: systemd
--
-- A start job for unit pvesr.service has finished with a failure.
--
-- The job identifier is 613367 and the job result is failed.
Aug 12 09:56:01 pve02 cron[2651]: (*system*vzdump) CAN'T OPEN SYMLINK (/etc/cron.d/vzdump)
Aug 12 09:56:04 pve02 pveproxy[221618]: worker exit
Aug 12 09:56:04 pve02 pveproxy[221619]: worker exit
Aug 12 09:56:04 pve02 pveproxy[221620]: worker exit
Aug 12 09:56:04 pve02 pveproxy[1008347]: worker 221619 finished
Aug 12 09:56:04 pve02 pveproxy[1008347]: worker 221620 finished
Aug 12 09:56:04 pve02 pveproxy[1008347]: worker 221618 finished
Aug 12 09:56:04 pve02 pveproxy[1008347]: starting 3 worker(s)
Aug 12 09:56:04 pve02 pveproxy[1008347]: worker 221717 started
Aug 12 09:56:04 pve02 pveproxy[1008347]: worker 221718 started
Aug 12 09:56:04 pve02 pveproxy[1008347]: worker 221719 started
Aug 12 09:56:04 pve02 pveproxy[221717]:
/etc/pve/local/pve-ssl.key: failed to load local private key (key_file or key) at /usr/share/perl5/PVE/APIServer/AnyEvent.pm line 1688. Aug 12 09:56:04 pve02 pveproxy[221718]:
/etc/pve/local/pve-ssl.key: failed to load local private key (key_file or key) at /usr/share/perl5/PVE/APIServer/AnyEvent.pm line 1688. Aug 12 09:56:04 pve02 pveproxy[221719]:
/etc/pve/local/pve-ssl.key: failed to load local private key (key_file or key) at /usr/share/perl5/PVE/APIServer/AnyEvent.pm line 1688. Aug 12 09:56:09 pve02 pveproxy[221717]: worker exit
Aug 12 09:56:09 pve02 pveproxy[221718]: worker exit
Aug 12 09:56:09 pve02 pveproxy[221719]: worker exit
Aug 12 09:56:09 pve02 pveproxy[1008347]: worker 221719 finished
Aug 12 09:56:09 pve02 pveproxy[1008347]: worker 221718 finished
Aug 12 09:56:09 pve02 pveproxy[1008347]: worker 221717 finished
Aug 12 09:56:09 pve02 pveproxy[1008347]: starting 3 worker(s)
Aug 12 09:56:09 pve02 pveproxy[1008347]: worker 221820 started
Aug 12 09:56:09 pve02 pveproxy[1008347]: worker 221821 started
Aug 12 09:56:09 pve02 pveproxy[1008347]: worker 221822 started
Aug 12 09:56:09 pve02 pveproxy[221820]:
/etc/pve/local/pve-ssl.key: failed to load local private key (key_file or key) at /usr/share/perl5/PVE/APIServer/AnyEvent.pm line 1688. Aug 12 09:56:09 pve02 pveproxy[221821]:
/etc/pve/local/pve-ssl.key: failed to load local private key (key_file or key) at /usr/share/perl5/PVE/APIServer/AnyEvent.pm line 1688. Aug 12 09:56:09 pve02 pveproxy[221822]:
/etc/pve/local/pve-ssl.key: failed to load local private key (key_file or key) at /usr/share/perl5/PVE/APIServer/AnyEvent.pm line 1688.
-----------------------------------
NODE 3
root@pve03:~# systemctl restart pve-cluster.service
root@pve03:~# systemctl status pve-cluster.service
pve-cluster.service - The Proxmox VE cluster filesystem
Loaded: loaded (/lib/systemd/system/pve-cluster.service; enabled; vendor preset: enabled)
Active:
active (running) since Mon 2019-08-12 10:00:49 EDT; 39s ago Process: 1211022 ExecStart=/usr/bin/pmxcfs (code=exited, status=0/SUCCESS)
Process: 1211124 ExecStartPost=/usr/bin/pvecm updatecerts --silent (code=exited, status=0/SUCCESS)
Main PID: 1211028 (pmxcfs)
Tasks: 6 (limit: 7372)
Memory: 15.9M
CGroup: /system.slice/pve-cluster.service
└─1211028 /usr/bin/pmxcfs
Aug 12 10:01:12 pve03 pmxcfs[1211028]:
[dcdb] crit: cpg_initialize failed: 2 Aug 12 10:01:12 pve03 pmxcfs[1211028]:
[status] crit: cpg_initialize failed: 2 Aug 12 10:01:18 pve03 pmxcfs[1211028]:
[quorum] crit: quorum_initialize failed: 2 Aug 12 10:01:18 pve03 pmxcfs[1211028]:
[confdb] crit: cmap_initialize failed: 2 Aug 12 10:01:18 pve03 pmxcfs[1211028]:
[dcdb] crit: cpg_initialize failed: 2 Aug 12 10:01:18 pve03 pmxcfs[1211028]:
[status] crit: cpg_initialize failed: 2 Aug 12 10:01:24 pve03 pmxcfs[1211028]:
[quorum] crit: quorum_initialize failed: 2 Aug 12 10:01:24 pve03 pmxcfs[1211028]:
[confdb] crit: cmap_initialize failed: 2 Aug 12 10:01:24 pve03 pmxcfs[1211028]:
[dcdb] crit: cpg_initialize failed: 2 Aug 12 10:01:24 pve03 pmxcfs[1211028]:
[status] crit: cpg_initialize failed: 2