Hi,
There is a bug in Proxmox v3. A system service can't be stopped or restarted using /etc/init.d script after service is upgraded. This happened on rsyslog service for us.
This bug affects all init scripts using start-stop-daemon with --exec argument. Let me show you an example.
Clean Debian 7 system:
mp@d7-tester:~$ uname -r
3.2.0-4-amd64
mp@d7-tester:~$ cat > file &
[1] 3216
mp@d7-tester:~$ rm file
[1]+ Stopped cat > file
mp@d7-tester:~$ ls -l /proc/3216/fd/
total 0
lrwx------. 1 mp mp 64 Nov 7 16:53 0 -> /dev/pts/0
l-wx------. 1 mp mp 64 Nov 7 16:53 1 -> /home/mp/file (deleted)
lrwx------. 1 mp mp 64 Nov 7 16:53 2 -> /dev/pts/0
This shows a process writing to a deleted file. The marker " (deleted)" is at the end of link data.
Now the proxmox:
mp@proxmox03:~$ uname -r
2.6.32-27-pve
mp@proxmox03:~$ cat > file &
[1] 897792
mp@proxmox03:~$ rm file
[1]+ Stopped cat > file
mp@proxmox03:~$ ls -l /proc/897792/fd/
total 0
lrwx------ 1 mp users 64 Nov 7 16:58 0 -> /dev/pts/1
l-wx------ 1 mp users 64 Nov 7 16:58 1 -> (deleted)/home/mp/file
lrwx------ 1 mp users 64 Nov 7 16:58 2 -> /dev/pts/1
Notice the marker " (deleted)" is at the beginning of link data.
Now the rsyslog is stopped in /etc/init.d/rsyslog by:
start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile /var/run/rsyslogd.pid --exec /usr/sbin/rsyslogd
Pid file exists with proper content, yet start-stop-daemon can't find the running process. I analysed its source code. It it provided in dpkg package. It turns out it expects the marker to be at the end of link. strace shows:
mp@proxmox03:~$ sudo strace -f start-stop-daemon --stop --retry=TERM/30/KILL/5 --pidfile /var/run/rsyslogd.pid --exec /usr/sbin/rsyslogd
[sudo] password for mp:
execve("/sbin/start-stop-daemon", ["start-stop-daemon", "--stop", "--retry=TERM/30/KILL/5", "--pidfile", "/var/run/rsyslogd.pid", "--exec", "/usr/sbin/rsyslogd"], [/* 17 vars */]) = 0
brk(0) = 0x1cd3000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852fc000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=29292, ...}) = 0
mmap(NULL, 29292, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc4852f4000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\357\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1603600, ...}) = 0
mmap(NULL, 3717176, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc484d53000
mprotect(0x7fc484ed5000, 2097152, PROT_NONE) = 0
mmap(0x7fc4850d5000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x182000) = 0x7fc4850d5000
mmap(0x7fc4850da000, 18488, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc4850da000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852f3000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852f2000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852f1000
arch_prctl(ARCH_SET_FS, 0x7fc4852f2700) = 0
mprotect(0x7fc4850d5000, 16384, PROT_READ) = 0
mprotect(0x605000, 4096, PROT_READ) = 0
mprotect(0x7fc4852fe000, 4096, PROT_READ) = 0
munmap(0x7fc4852f4000, 29292) = 0
brk(0) = 0x1cd3000
brk(0x1cf4000) = 0x1cf4000
stat("/usr/sbin/rsyslogd", {st_mode=S_IFREG|0755, st_size=384416, ...}) = 0
open("/var/run/rsyslogd.pid", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=7, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852fb000
read(3, "423994\n", 4096) = 7
readlink("/proc/423994/exe", " (deleted)/usr/sbin/rsyslogd"..., 256) = 28
stat(" (deleted)/usr/sbin/rsyslogd", 0x7fff7938a770) = -1 ENOENT (No such file or directory)
close(3) = 0
munmap(0x7fc4852fb000, 4096) = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852fb000
write(1, "No /usr/sbin/rsyslogd found runn"..., 50No /usr/sbin/rsyslogd found running; none killed.
) = 50
exit_group(1) = ?
start-stop-daemon invokes stat with improper path - stat(" (deleted)/usr/sbin/rsyslogd", 0x7fff7938a770)
Therefore I urge proxmox developers to fix the bug by patching the kernel so it matches version 3.2.0 or change start-stop-daemon code to old behavior.
Please post any comment if you intend to fix the bug.
There is a bug in Proxmox v3. A system service can't be stopped or restarted using /etc/init.d script after service is upgraded. This happened on rsyslog service for us.
This bug affects all init scripts using start-stop-daemon with --exec argument. Let me show you an example.
Clean Debian 7 system:
mp@d7-tester:~$ uname -r
3.2.0-4-amd64
mp@d7-tester:~$ cat > file &
[1] 3216
mp@d7-tester:~$ rm file
[1]+ Stopped cat > file
mp@d7-tester:~$ ls -l /proc/3216/fd/
total 0
lrwx------. 1 mp mp 64 Nov 7 16:53 0 -> /dev/pts/0
l-wx------. 1 mp mp 64 Nov 7 16:53 1 -> /home/mp/file (deleted)
lrwx------. 1 mp mp 64 Nov 7 16:53 2 -> /dev/pts/0
This shows a process writing to a deleted file. The marker " (deleted)" is at the end of link data.
Now the proxmox:
mp@proxmox03:~$ uname -r
2.6.32-27-pve
mp@proxmox03:~$ cat > file &
[1] 897792
mp@proxmox03:~$ rm file
[1]+ Stopped cat > file
mp@proxmox03:~$ ls -l /proc/897792/fd/
total 0
lrwx------ 1 mp users 64 Nov 7 16:58 0 -> /dev/pts/1
l-wx------ 1 mp users 64 Nov 7 16:58 1 -> (deleted)/home/mp/file
lrwx------ 1 mp users 64 Nov 7 16:58 2 -> /dev/pts/1
Notice the marker " (deleted)" is at the beginning of link data.
Now the rsyslog is stopped in /etc/init.d/rsyslog by:
start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile /var/run/rsyslogd.pid --exec /usr/sbin/rsyslogd
Pid file exists with proper content, yet start-stop-daemon can't find the running process. I analysed its source code. It it provided in dpkg package. It turns out it expects the marker to be at the end of link. strace shows:
mp@proxmox03:~$ sudo strace -f start-stop-daemon --stop --retry=TERM/30/KILL/5 --pidfile /var/run/rsyslogd.pid --exec /usr/sbin/rsyslogd
[sudo] password for mp:
execve("/sbin/start-stop-daemon", ["start-stop-daemon", "--stop", "--retry=TERM/30/KILL/5", "--pidfile", "/var/run/rsyslogd.pid", "--exec", "/usr/sbin/rsyslogd"], [/* 17 vars */]) = 0
brk(0) = 0x1cd3000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852fc000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=29292, ...}) = 0
mmap(NULL, 29292, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fc4852f4000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\357\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1603600, ...}) = 0
mmap(NULL, 3717176, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fc484d53000
mprotect(0x7fc484ed5000, 2097152, PROT_NONE) = 0
mmap(0x7fc4850d5000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x182000) = 0x7fc4850d5000
mmap(0x7fc4850da000, 18488, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fc4850da000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852f3000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852f2000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852f1000
arch_prctl(ARCH_SET_FS, 0x7fc4852f2700) = 0
mprotect(0x7fc4850d5000, 16384, PROT_READ) = 0
mprotect(0x605000, 4096, PROT_READ) = 0
mprotect(0x7fc4852fe000, 4096, PROT_READ) = 0
munmap(0x7fc4852f4000, 29292) = 0
brk(0) = 0x1cd3000
brk(0x1cf4000) = 0x1cf4000
stat("/usr/sbin/rsyslogd", {st_mode=S_IFREG|0755, st_size=384416, ...}) = 0
open("/var/run/rsyslogd.pid", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=7, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852fb000
read(3, "423994\n", 4096) = 7
readlink("/proc/423994/exe", " (deleted)/usr/sbin/rsyslogd"..., 256) = 28
stat(" (deleted)/usr/sbin/rsyslogd", 0x7fff7938a770) = -1 ENOENT (No such file or directory)
close(3) = 0
munmap(0x7fc4852fb000, 4096) = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fc4852fb000
write(1, "No /usr/sbin/rsyslogd found runn"..., 50No /usr/sbin/rsyslogd found running; none killed.
) = 50
exit_group(1) = ?
start-stop-daemon invokes stat with improper path - stat(" (deleted)/usr/sbin/rsyslogd", 0x7fff7938a770)
Therefore I urge proxmox developers to fix the bug by patching the kernel so it matches version 3.2.0 or change start-stop-daemon code to old behavior.
Please post any comment if you intend to fix the bug.
Last edited:
