After POWER FAILURE on PROXMOX, TURNKEY-UBUNTU CT had a strange behavior

I

italian01

Member
Feb 23, 2012
57
0
6
Italy
Hello,

there was a power failure last week-end and, therefore, a brute shutdown of my Proxmox Server. There were (and are) 4 Container in that server: 3 running and 1 not. The first 3 virtual machines are 2 Windows 2003 Servers and 1 is an Ubuntu TurnKey LAMP 11.3. The last one virtual machine not running is another Ubuntu TurnKey LAMP 11.3.

When I turned on the Proxmox Server, all VZ works fine (also that which is usually not running) except the Ubuntu TurnKey LAMP 11.3 which is usually running. This VZ present a very strange behavior which I can summarize in short by listing 3 not-working service:

1. I cannot login into a root home;
2. There isn't history and I lose it at each logout;
3. Network is always down unless I run "service networking start" manually.

What can be happened?

Note that I did each tests by logging in VZ with "vzctl enter" command.


Some infos:

**** ON THE PROXMOX ****

root@proxmox2:/var/lib/vz/root/101.mnt# pveversion -v
pve-manager: 2.3-13 (pve-manager/2.3/7946f1f1)
running kernel: 2.6.32-19-pve
proxmox-ve-2.6.32: 2.3-96
pve-kernel-2.6.32-11-pve: 2.6.32-66
pve-kernel-2.6.32-19-pve: 2.6.32-96
lvm2: 2.02.95-1pve2
clvm: 2.02.95-1pve2
corosync-pve: 1.4.4-4
openais-pve: 1.1.4-2
libqb: 0.10.1-2
redhat-cluster-pve: 3.1.93-2
resource-agents-pve: 3.9.2-3
fence-agents-pve: 3.1.9-1
pve-cluster: 1.0-36
qemu-server: 2.3-20
pve-firmware: 1.0-21
libpve-common-perl: 1.0-49
libpve-access-control: 1.0-26
libpve-storage-perl: 2.3-7
vncterm: 1.0-4
vzctl: 4.0-1pve2
vzprocps: 2.0.11-2
vzquota: 3.1-1
pve-qemu-kvm: 1.4-10
ksm-control-daemon: 1.1-1


**** ON THE VZ CT *******

root@VMLAMP01 /# uname -a
Linux VMLAMP01 2.6.32-19-pve #1 SMP Wed May 15 07:32:52 CEST 2013 i686 GNU/Linux

root@VMLAMP01 /# mount
[mntent]: line 5 in /etc/mtab is bad
[mntent]: line 7 in /etc/mtab is bad
[mntent]: line 8 in /etc/mtab is bad
/dev/simfs on / type simfs (rw,relatime)
//fs1/datipos/ on /mnt/www/pos/content type cifs (ro,mand,relatime,sec=ntlm,unc=\\fs1\datipos,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536)
//fs1/o_d_g/ on /mnt/www/pos/odg type cifs (ro,mand,relatime,sec=ntlm,unc=\\fs1\o_d_g,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536)
//fs1/ufficiosevizi/ on /mnt/www/pos/uffser type cifs (ro,mand,relatime,sec=ntlm,unc=\\fs1\ufficiosevizi,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536)
//fs1/personale/ on /mnt/www/pos/uffper type cifs (ro,mand,relatime,sec=ntlm,unc=\\fs1\personale,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
varrun on /var/run type tmpfs (rw,relatime)
varlock on /var/lock type tmpfs (rw,relatime)
tmpfs on /run type tmpfs (rw,nosuid,noexec,relatime,size=104860k,mode=755)
tmpfs on /run/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)
tmpfs on /run/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=419420k)
tmpfs on /var/run type tmpfs (rw,nosuid,noexec,relatime,size=104860k,mode=755)
tmpfs on /var/lock type tmpfs (rw,nosuid,nodev,noexec,relatime,size=5120k)

root@VMLAMP01 /# cat /etc/fstab
# UNCONFIGURED FSTAB FOR BASE SYSTEM

//fs1/datipos /mnt/www/pos/content cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0
//fs1/o_d_g /mnt/www/pos/odg cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0
//fs1/ufficiosevizi /mnt/www/pos/uffser cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0
//fs1/ufficio\040sicurezza /mnt/www/pos/uffsic cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0
//fs1/personale /mnt/www/pos/uffper cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0
//fs1/sala\040operativa /mnt/www/pos/salopa cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0
//fs1/sala\040operativa1 /mnt/www/pos/salopb cifs noauto,ro,credentials=/root/www_pos_fs1,iocharset=utf8,uid=www-data,gid=www-data 0 0

root@VMLAMP01 /# cat /etc/mtab
/dev/simfs / simfs rw,relatime 0 0
//fs1/datipos/ /mnt/www/pos/content cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\datipos,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
//fs1/o_d_g/ /mnt/www/pos/odg cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\o_d_g,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
//fs1/ufficiosevizi/ /mnt/www/pos/uffser cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\ufficiosevizi,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
//fs1/ufficio\040sicurezza/ /mnt/www/pos/uffsic cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\ufficio sicurezza,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
//fs1/personale/ /mnt/www/pos/uffper cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\personale,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
//fs1/sala\040operativa/ /mnt/www/pos/salopa cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\sala operativa,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
//fs1/sala\040operativa1/ /mnt/www/pos/salopb cifs ro,mand,relatime,sec=ntlm,unc=\\fs1\sala operativa1,username=www_pos,domain=TS,uid=33,forceuid,gid=33,forcegid,addr=192.168.2.15,file_mode=0755,dir_mode=0755,serverino,rsize=16384,wsize=65536 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
varrun /var/run tmpfs rw,relatime 0 0
varlock /var/lock tmpfs rw,relatime 0 0
tmpfs /run tmpfs rw,nosuid,noexec,relatime,size=104860k,mode=755 0 0
tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0
tmpfs /run/shm tmpfs rw,nosuid,nodev,noexec,relatime,size=419420k 0 0
tmpfs /var/run tmpfs rw,nosuid,noexec,relatime,size=104860k,mode=755 0 0
tmpfs /var/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0

root@VMLAMP01 /# service --status-all
[ ? ] acpid
[ + ] apache2
[ - ] bootlogd
[ - ] bootlogs
[ ? ] bootmisc.sh
[ - ] casper
[ ? ] checkfs.sh
[ ? ] checkroot-bootclean.sh
[ - ] checkroot.sh
[ ? ] confconsole
[ - ] console-setup
[ ? ] cron
[ ? ] di-live
[ ? ] failsafe-x
[ - ] grub-common
[ ? ] hostname
[ - ] hostname.sh
[ ? ] hubdns
[ ? ] hwclock.sh
[ ? ] inithooks
[ ? ] jboss5
[ + ] jetty
[ - ] keyboard-setup
[ ? ] killprocs
[ ? ] module-init-tools
[ - ] motd
[ ? ] mountall-bootclean.sh
[ ? ] mountall.sh
[ ? ] mountdevsubfs.sh
[ ? ] mountkernfs.sh
[ ? ] mountnfs-bootclean.sh
[ ? ] mountnfs.sh
[ ? ] mtab.sh
[ ? ] mysql
[ ? ] networking
[ + ] ntp
[ ? ] ondemand
[ ? ] plymouth
[ ? ] plymouth-log
[ ? ] plymouth-splash
[ ? ] plymouth-stop
[ - ] postfix
[ ? ] procps
[ + ] quota
[ - ] quotarpc
[ ? ] rc.local
[ - ] rmnologin
[ - ] rsync
[ ? ] rsyslog
[ - ] rsyslog.dpkg-new
[ ? ] screen-cleanup
[ ? ] sendsigs
[ + ] shellinabox
[ + ] ssh
[ ? ] stop-bootlogd
[ ? ] stop-bootlogd-single
[ ? ] udev
[ ? ] udev-finish
[ ? ] udevmonitor
[ ? ] udevtrigger
[ ? ] umountfs
[ ? ] umountnfs.sh
[ ? ] umountroot
[ - ] urandom
[ ? ] webmin
[ - ] winbind
[ - ] x11-common


Thank's in advance.
 
Last edited:
I made a breakthrogh.

So, the power failure isn't the cause of starup issues described so far: the power failure has just highlighted it. Instead, the cause it set on automatic upgrade of Turnkey LAMP.

After many researches and a lot of "googling", I came to the conclusion that the cause is on automatic upgrade of Turnkey LAMP which has likely modified its "UpStart" system; and, probably, I faced this issue just on the last power failure because that container has never rebooted before last upgrade. I came to this conclusion because it has been sufficient to add a sleep of 2 seconds in the networking.conf file for getting network interfaces automatically up since booting of this container.

But, it's not an asynchronous booting mechanism what I want!! This is a work-around, found by an ampirical method and I like to know where is the actual cause. Hence, I'm still here to ask help.

Follow some configuration files of this conteiner which could help.

Code: 
root@proxmox2:~# vzctl enter 101
entered into CT 101

root@VMLAMP01 /# ls
0  aquota.group  aquota.user  bin  boot  dev  etc  fastboot  home  lib  media  mnt  opt  proc  root  run  sbin  selinux  srv  sys  tmp  usr  var

root@VMLAMP01 /# ifconfig

root@VMLAMP01 /#

root@VMLAMP01 /# ifconfig -a
lo        Link encap:Local Loopback
          LOOPBACK  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

venet0    Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
          BROADCAST POINTOPOINT NOARP  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

root@VMLAMP01 /var/log# cat syslog | sed -n '/16:24/,$ p' | egrep -vi '(ntp|mysq|cro|post)'
Feb 27 16:24:25 VMLAMP01 kernel: imklog 5.8.11, log source = /proc/kmsg started.
Feb 27 16:24:25 VMLAMP01 rsyslogd: [origin software="rsyslogd" swVersion="5.8.11" x-pid="80" x-info="http://www.rsyslog.com"] start
Feb 27 16:24:25 VMLAMP01 kernel: imklog 5.8.11, log source = /proc/kmsg started.
Feb 27 16:24:25 VMLAMP01 rsyslogd: [origin software="rsyslogd" swVersion="5.8.11" x-pid="80" x-info="http://www.rsyslog.com"] start
Feb 27 16:24:25 VMLAMP01 init: Failed to spawn smbd main process: unable to execute: No such file or directory
Feb 27 16:24:25 VMLAMP01 init: Failed to spawn smbd main process: unable to execute: No such file or directory
Feb 27 16:24:25 VMLAMP01 init: Failed to spawn smbd main process: unable to execute: No such file or directory
Feb 27 16:24:25 VMLAMP01 init: Failed to spawn smbd main process: unable to execute: No such file or directory
Feb 27 16:24:25 VMLAMP01 init: rc-sysinit main process (74) killed by TERM signal
Feb 27 16:24:25 VMLAMP01 init: rc-sysinit main process (74) killed by TERM signal
Feb 27 16:24:26 VMLAMP01 acpid: cannot open input layer
Feb 27 16:24:26 VMLAMP01 acpid: cannot open input layer
Feb 27 16:24:26 VMLAMP01 acpid: inotify_add_watch() failed: No such file or directory (2)
Feb 27 16:24:26 VMLAMP01 acpid: inotify_add_watch() failed: No such file or directory (2)
Feb 27 16:24:26 VMLAMP01 acpid: starting up with netlink and the input layer
Feb 27 16:24:26 VMLAMP01 acpid: starting up with netlink and the input layer
Feb 27 16:24:26 VMLAMP01 acpid: 1 rule loaded
Feb 27 16:24:26 VMLAMP01 acpid: 1 rule loaded
Feb 27 16:24:26 VMLAMP01 acpid: waiting for events: event logging is off
Feb 27 16:24:26 VMLAMP01 acpid: waiting for events: event logging is off

root@VMLAMP01 /var/log# ls /etc/init
cron.conf               mountall.conf                     network-interface.conf  openvz.conf      rsyslog.conf         ssh.conf.dpkg-dist
failsafe-x.conf         mysql.conf                        networking.conf         procps.conf      screen-cleanup.conf  udev-fallback-graphics.conf
hostname.conf           network-interface-container.conf  networking.conf.old     rc-sysinit.conf  smbd.conf            udev.conf
module-init-tools.conf  network-interface-security.conf   nmbd.conf               rc.conf          ssh.conf             udevtrigger.conf

Code: 
root@VMLAMP01 /var/log# cat /etc/init/openvz.conf
# OpenVZ - Fix init sequence to have OpenVZ working with upstart
# Source: [URL]http://blog.bodhizazen.net/linux/ubuntu-10-04-openvz-templates/[/URL]

description "Fix OpenVZ"

start on startup

task

pre-start script

 mount -t devpts devpts /dev/pts
 mount -t tmpfs varrun /var/run
 mount -t tmpfs varlock /var/lock
 mkdir -p /var/run/network

 if [ ! -e /etc/mtab ]; then
  cat /proc/mounts > /etc/mtab
 fi

 touch /var/run/utmp
 chmod 664 /var/run/utmp
 chown root.utmp /var/run/utmp

 if [ "$(find /etc/network/ -name upstart -type f)" ]; then
  chmod -x /etc/network/*/upstart || true
 fi

end script

script

 start networking
 initctl emit filesystem --no-wait
 initctl emit local-filesystems --no-wait
 initctl emit virtual-filesystems --no-wait
 initctl emit net-device-up --no-wait
 init 2

end script
Code: 
root@VMLAMP01 /var/log# cat /etc/init/mountall.conf
# mountall - Mount filesystems on boot
#
# This helper mounts filesystems in the correct order as the devices
# and mountpoints become available.

description "Mount filesystems on boot"

start on startup

task

emits virtual-filesystems
emits local-filesystems
emits remote-filesystems
emits all-swaps
emits all-filesystems
emits filesystem

pre-start script

    find /var/run -mindepth 1 -maxdepth 1 | grep -v utmp | xargs rm -rf
    mkdir -p /var/run/network
    find /var/lock -mindepth 1 -maxdepth 1 | xargs rm -rf

end script

post-start script
    initctl emit -n filesystem
    initctl emit -n all-swaps
    initctl emit -n all-filesystems
    initctl emit -n virtual-filesystems
    initctl emit -n remote-filesystems
    initctl emit -n local-filesystems
    mount -a
end script

Code: 
root@VMLAMP01 /var/log# cat /etc/init/networking.conf
# networking - configure virtual network devices
#
# This task causes virtual network devices that do not have an associated
# kernel object to be started on boot.

description "configure virtual network devices"

emits static-network-up
emits net-device-up

start on local-filesystems
stop on unmounted-remote-filesystems

pre-start script

#    sleep 2
    mkdir -p /run/network
    ifup -a

end script

post-stop script
    log_warning_msg() {
        echo $*
    }

    # These checks were taken from the Debian ifupdown.networking.init script
    check_network_file_systems() {
        [ -e /proc/mounts ] || return 0

        if [ -e /etc/iscsi/iscsi.initramfs ]; then
            log_warning_msg "not deconfiguring network interfaces: iSCSI root is mounted."
            exit 0
        fi

        while read DEV MTPT FSTYPE REST; do
            case $DEV in
            /dev/nbd*|/dev/nd[a-z]*|/dev/etherd/e*)
                log_warning_msg "not deconfiguring network interfaces: network devices still mounted."
                exit 0
                ;;
            esac
            case $FSTYPE in
            nfs|nfs4|smbfs|ncp|ncpfs|cifs|coda|ocfs2|gfs|pvfs|pvfs2|fuse.httpfs|fuse.curlftpfs)
                log_warning_msg "not deconfiguring network interfaces: network file systems still mounted."
                exit 0
                ;;
            esac
        done < /proc/mounts
    }

    check_network_swap() {
        [ -e /proc/swaps ] || return 0

        while read DEV MTPT FSTYPE REST; do
            case $DEV in
            /dev/nbd*|/dev/nd[a-z]*|/dev/etherd/e*)
                log_warning_msg "not deconfiguring network interfaces: network swap still mounted."
                exit 0
                ;;
            esac
        done < /proc/swaps
    }

    check_network_file_systems
    check_network_swap
    ifdown -a
end script
Code: 
root@VMLAMP01 /var/log# start networking
start: Job is already running: networking

root@VMLAMP01 /var/log# ifconfig

root@VMLAMP01 /var/log# stop networking
networking stop/waiting

root@VMLAMP01 /var/log# start networking
networking start/running

root@VMLAMP01 /var/log# ifconfig
lo        Link encap:Local Loopback
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)

venet0    Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
          inet addr:127.0.0.2  P-t-P:127.0.0.2  Bcast:0.0.0.0  Mask:255.255.255.255
          UP BROADCAST POINTOPOINT RUNNING NOARP  MTU:1500  Metric:1
          RX packets:1 errors:0 dropped:0 overruns:0 frame:0
          TX packets:4 errors:0 dropped:4 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:124 (124.0 B)  TX bytes:240 (240.0 B)

venet0:0  Link encap:UNSPEC  HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00
          inet addr:192.168.2.13  P-t-P:192.168.2.13  Bcast:192.168.2.13  Mask:255.255.255.255
          UP BROADCAST POINTOPOINT RUNNING NOARP  MTU:1500  Metric:1

Help me, please|||||

Thank's in advance.
 
You must log in or register to reply here.
Top Bottom