Hello!
I've a cluster of 3 servers running PVE 3.0 up to date. The server are Dell R420. I've 13 openvz CT distributed on the 3 servers.
I've setup ONE backup rule to backup each night all CT from the 3 nodes (SNAPHOST + LZO on a NAS using NFS). Some day, the backup crash during the night, the CT of the crashed node are not longer accessibles and the load on the node is very very high (this night during the last crash, it was 1350).
The crash is not alway on the same node and it was impossible to run the system for more one week without a crash.
The crashed node is still accessible using ssh but the CT can't be stopped (from shell or from GUI). A reboot command on the node don't work (do nothing). I have to do a hardware reset.
It seem that the crash arrive at the end of the node backup (the last words message is when the system say "rm" the previous snapshot of the last CT on the node).
I've found this in the node with dmesg:
INFO: task flush-253:2:1786 blocked for more than 120 seconds."echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
flush-253:2 D ffff880829b42080 0 1786 2 0 0x00000000
ffff8808285417b0 0000000000000046 ffff880828541710 ffff880c32833900
ffff880828541770 ffffffff8140b5ec ffff880828541740 ffff88102a657c00
ffff880ba1611a40 0000000000000286 ffff880828541fd8 ffff880828541fd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff811c77a0>] sync_buffer+0x40/0x50
[<ffffffff8151c31b>] __wait_on_bit_lock+0x5b/0xc0
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151c3fc>] out_of_line_wait_on_bit_lock+0x7c/0x90
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff811c92f0>] ? end_buffer_async_write+0x0/0x180
[<ffffffff811c8136>] __lock_buffer+0x36/0x40
[<ffffffff811c9e85>] __block_write_full_page+0x185/0x4e0
[<ffffffff81045550>] ? flush_tlb_others_ipi+0x120/0x130
[<ffffffff811c92f0>] ? end_buffer_async_write+0x0/0x180
[<ffffffff811c76f0>] ? generic_submit_bh_handler+0x0/0x10
[<ffffffff8126f250>] ? prio_tree_next+0x70/0x170
[<ffffffff81045658>] ? flush_tlb_page+0x48/0xb0
[<ffffffff811ca686>] generic_block_write_full_page+0x106/0x110
[<ffffffff811c92f0>] ? end_buffer_async_write+0x0/0x180
[<ffffffff811c76f0>] ? generic_submit_bh_handler+0x0/0x10
[<ffffffff811ca6a8>] block_write_full_page_endio+0x18/0x20
[<ffffffff811ca6c5>] block_write_full_page+0x15/0x20
[<ffffffffa009a3dd>] ext3_ordered_writepage+0x1ed/0x240 [ext3]
[<ffffffff81137887>] __writepage+0x17/0x40
[<ffffffff811380b6>] write_cache_pages+0x1f6/0x440
[<ffffffff81137870>] ? __writepage+0x0/0x40
[<ffffffff81138324>] generic_writepages+0x24/0x30
[<ffffffff81138dbd>] do_writepages+0x3d/0x50
[<ffffffff811be472>] __writeback_single_inode+0xb2/0x280
[<ffffffff811be6c3>] writeback_single_inode+0x83/0xc0
[<ffffffff811ae370>] ? iput+0x30/0x70
[<ffffffff811bf233>] writeback_sb_inodes+0x103/0x1e0
[<ffffffff811bf45f>] writeback_inodes_wb+0xff/0x170
[<ffffffff811bf78b>] wb_writeback+0x2bb/0x3f0
[<ffffffff811bfa46>] wb_do_writeback+0x186/0x240
[<ffffffff8107b900>] ? process_timeout+0x0/0x10
[<ffffffff811bfb5b>] bdi_writeback_task+0x5b/0x1b0
[<ffffffff81094fa7>] ? bit_waitqueue+0x17/0xc0
[<ffffffff8114d4d2>] bdi_start_fn+0x92/0x100
[<ffffffff8114d440>] ? bdi_start_fn+0x0/0x100
[<ffffffff81094d68>] kthread+0x88/0x90
[<ffffffff810096d2>] ? __switch_to+0xc2/0x2f0
[<ffffffff8100c22a>] child_rip+0xa/0x20
[<ffffffff81094ce0>] ? kthread+0x0/0x90
[<ffffffff8100c220>] ? child_rip+0x0/0x20
INFO: task kjournald:2087 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
kjournald D ffff8808292581c0 0 2087 2 0 0x00000000
ffff8808287a5c70 0000000000000046 ffff8808287a5c34 ffff880c32833900
ffff8808287a5c30 ffffffff8140b5ec ffff8808287a5c20 ffffffff8109f5b3
0000000000000000 0000000000000282 ffff8808287a5fd8 ffff8808287a5fd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff8109f5b3>] ? ktime_get_ts+0xb3/0xf0
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff811c77a0>] sync_buffer+0x40/0x50
[<ffffffff8151c470>] __wait_on_bit+0x60/0x90
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151c51c>] out_of_line_wait_on_bit+0x7c/0x90
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff811c80f6>] __wait_on_buffer+0x26/0x30
[<ffffffffa0071100>] journal_commit_transaction+0xc00/0x1200 [jbd]
[<ffffffff8107c8d8>] ? lock_timer_base.isra.76+0x38/0x70
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff8107e3c4>] ? try_to_del_timer_sync+0x84/0xe0
[<ffffffffa0076d05>] kjournald+0xe5/0x230 [jbd]
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffffa0076c20>] ? kjournald+0x0/0x230 [jbd]
[<ffffffff81094d68>] kthread+0x88/0x90
[<ffffffff810096d2>] ? __switch_to+0xc2/0x2f0
[<ffffffff8100c22a>] child_rip+0xa/0x20
[<ffffffff81094ce0>] ? kthread+0x0/0x90
[<ffffffff8100c220>] ? child_rip+0x0/0x20
INFO: task mysqld:7229 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
mysqld D ffff881016295020 0 7229 7032 115 0x00000000
ffff88102a685ce8 0000000000000082 ffff88102a685cac ffffffff8140b5ec
ffff88102a685ca8 ffffffff8160bbc0 0000000000956e68 ffff88102c42c810
0000000000000000 0000000165739904 ffff88102a685fd8 ffff88102a685fd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff8111fe90>] ? sync_page+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff8111fecb>] sync_page+0x3b/0x50
[<ffffffff8151c470>] __wait_on_bit+0x60/0x90
[<ffffffff81120060>] wait_on_page_bit+0x80/0x90
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff8112039a>] wait_on_page_writeback_range.part.36+0xea/0x180
[<ffffffff81138324>] ? generic_writepages+0x24/0x30
[<ffffffff81138dbd>] ? do_writepages+0x3d/0x50
[<ffffffff81120cbb>] ? __filemap_fdatawrite_range+0x5b/0x60
[<ffffffff81120d45>] wait_on_page_writeback_range+0x15/0x20
[<ffffffff81120dbf>] filemap_write_and_wait_range+0x6f/0x80
[<ffffffff811c4333>] vfs_fsync_range+0xa3/0x180
[<ffffffff811c447d>] vfs_fsync+0x1d/0x20
[<ffffffff811c46a6>] do_fsync+0x66/0xa0
[<ffffffff811c4e40>] sys_fsync+0x10/0x20
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task apache2:579195 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
apache2 D ffff8803370f8540 0 579195 553190 115 0x00000000
ffff88065ed6da78 0000000000000086 0000000000000000 ffffffff8140b5ec
0000000000000004 0000000000000001 0000000000000001 0000000000000282
ffff88065ed6da28 0000000165739251 ffff88065ed6dfd8 ffff88065ed6dfd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff8111fe90>] ? sync_page+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff8111fecb>] sync_page+0x3b/0x50
[<ffffffff8151c31b>] __wait_on_bit_lock+0x5b/0xc0
[<ffffffff8111fe67>] __lock_page+0x67/0x70
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff811c2bff>] __generic_file_splice_read+0x54f/0x720
[<ffffffff811c1460>] ? spd_release_page+0x0/0x20
[<ffffffff81435ec4>] ? sock_aio_write+0x124/0x160
[<ffffffff811ac539>] ? dput_nocache+0xc9/0x250
[<ffffffff81435da0>] ? sock_aio_write+0x0/0x160
[<ffffffff8119378a>] ? do_sync_readv_writev+0xfa/0x140
[<ffffffff811c2e0e>] generic_file_splice_read+0x3e/0x90
[<ffffffff811c13db>] do_splice_to+0x7b/0xa0
[<ffffffff811c1c57>] splice_direct_to_actor+0xb7/0x1d0
[<ffffffff811c1330>] ? direct_splice_actor+0x0/0x30
[<ffffffff811c1dc1>] do_splice_direct+0x51/0x70
[<ffffffff81193084>] do_sendfile+0x174/0x200
[<ffffffff81194067>] sys_sendfile64+0x57/0xa0
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task apache2:580533 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
apache2 D ffff8802059ce880 0 580533 553190 115 0x00000000
ffff88033704bcd8 0000000000000082 0000000000000000 0000000000000000
0000000000001f40 fffffff500000001 0000020000000000 ffff8802cf9e2240
00007f713f2481b0 000000016573b37a ffff88033704bfd8 ffff88033704bfd8
Call Trace:
[<ffffffff8151cb62>] __mutex_lock_slowpath+0x122/0x170
[<ffffffff8151c57a>] mutex_lock+0x2a/0x50
[<ffffffff811216f1>] generic_file_aio_write+0x71/0x100
[<ffffffff81192a1e>] do_sync_write+0xfe/0x140
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81194bd9>] ? __fput+0x199/0x220
[<ffffffff811931b1>] vfs_write+0xa1/0x190
[<ffffffff8119350a>] sys_write+0x4a/0x90
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task apache2:580840 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
apache2 D ffff88082848ad60 0 580840 553190 115 0x00000000
ffff880b0abe3938 0000000000000086 0000000000000000 ffff880c32833900
ffff880b0abe38f8 ffffffff8140b5ec 0000000000000050 ffff88000002de40
0000000000000002 000000016573a206 ffff880b0abe3fd8 ffff880b0abe3fd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff811c77a0>] sync_buffer+0x40/0x50
[<ffffffff8151c31b>] __wait_on_bit_lock+0x5b/0xc0
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151c3fc>] out_of_line_wait_on_bit_lock+0x7c/0x90
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff811c8136>] __lock_buffer+0x36/0x40
[<ffffffff811c87a5>] __sync_dirty_buffer+0x35/0xe0
[<ffffffff811c8863>] sync_dirty_buffer+0x13/0x20
[<ffffffffa006f97a>] journal_dirty_data+0x19a/0x230 [jbd]
[<ffffffffa009adc0>] ext3_journal_dirty_data+0x20/0x50 [ext3]
[<ffffffffa009ae15>] journal_dirty_data_fn+0x25/0x30 [ext3]
[<ffffffffa0099a6d>] walk_page_buffers+0x8d/0xd0 [ext3]
[<ffffffffa009adf0>] ? journal_dirty_data_fn+0x0/0x30 [ext3]
[<ffffffffa009e2bb>] ext3_ordered_write_end+0xab/0x1c0 [ext3]
[<ffffffff81120f45>] generic_file_buffered_write_iter+0x175/0x2a0
[<ffffffff811213e8>] __generic_file_write_iter+0x188/0x380
[<ffffffff81121663>] __generic_file_aio_write+0x83/0xa0
[<ffffffff81121708>] generic_file_aio_write+0x88/0x100
[<ffffffff81192a1e>] do_sync_write+0xfe/0x140
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811ac539>] ? dput_nocache+0xc9/0x250
[<ffffffff811b4325>] ? mntput_no_expire+0x25/0xc0
[<ffffffff81194bd9>] ? __fput+0x199/0x220
[<ffffffff811931b1>] vfs_write+0xa1/0x190
[<ffffffff8119350a>] sys_write+0x4a/0x90
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task apache2:580873 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
apache2 D ffff8802059cf360 0 580873 553190 115 0x00000000
ffff880dc12d7938 0000000000000082 0000000000000004 ffff880c32833900
ffff880dc12d78f8 ffffffff8140b5ec 0000000000000050 ffff88000002de40
0000000000000002 0000000000000286 ffff880dc12d7fd8 ffff880dc12d7fd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff811c77a0>] sync_buffer+0x40/0x50
[<ffffffff8151c31b>] __wait_on_bit_lock+0x5b/0xc0
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151c3fc>] out_of_line_wait_on_bit_lock+0x7c/0x90
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff811c8136>] __lock_buffer+0x36/0x40
[<ffffffff811c87a5>] __sync_dirty_buffer+0x35/0xe0
[<ffffffff811c8863>] sync_dirty_buffer+0x13/0x20
[<ffffffffa006f97a>] journal_dirty_data+0x19a/0x230 [jbd]
[<ffffffffa009adc0>] ext3_journal_dirty_data+0x20/0x50 [ext3]
[<ffffffffa009ae15>] journal_dirty_data_fn+0x25/0x30 [ext3]
[<ffffffffa0099a6d>] walk_page_buffers+0x8d/0xd0 [ext3]
[<ffffffffa009adf0>] ? journal_dirty_data_fn+0x0/0x30 [ext3]
[<ffffffffa009e2bb>] ext3_ordered_write_end+0xab/0x1c0 [ext3]
[<ffffffff81120f45>] generic_file_buffered_write_iter+0x175/0x2a0
[<ffffffff811213e8>] __generic_file_write_iter+0x188/0x380
[<ffffffff81121663>] __generic_file_aio_write+0x83/0xa0
[<ffffffff81121708>] generic_file_aio_write+0x88/0x100
[<ffffffff81192a1e>] do_sync_write+0xfe/0x140
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811931b1>] vfs_write+0xa1/0x190
[<ffffffff8119350a>] sys_write+0x4a/0x90
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task lvremove:581374 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
lvremove D ffff8802ed7761c0 0 581374 553330 0 0x00000000
ffff880f9b8a3a88 0000000000000082 ffffea00275a96c0 ffff880c32833900
ffff880f9b8a3a48 ffffffff8140b5ec 0000000000000008 0000000000001000
0000000000000000 00000001657393ee ffff880f9b8a3fd8 ffff880f9b8a3fd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff811d1bcc>] __blockdev_direct_IO_newtrunc+0xa8c/0xce0
[<ffffffff81120346>] ? wait_on_page_writeback_range.part.36+0x96/0x180
[<ffffffff811d1e7c>] __blockdev_direct_IO+0x5c/0xd0
[<ffffffff811cec90>] ? blkdev_get_blocks+0x0/0xd0
[<ffffffff811cef07>] blkdev_direct_IO+0x57/0x60
[<ffffffff811cec90>] ? blkdev_get_blocks+0x0/0xd0
[<ffffffff811200b8>] mapping_direct_IO.isra.25+0x48/0x70
[<ffffffff81123542>] generic_file_read_iter+0x622/0x6a0
[<ffffffff8126c817>] ? kobject_put+0x27/0x60
[<ffffffff8140a710>] ? dm_blk_open+0x70/0x80
[<ffffffff811cfd69>] ? __blkdev_get+0x1b9/0x3e0
[<ffffffff8112365f>] generic_file_aio_read+0x9f/0xb0
[<ffffffff811cef52>] blkdev_aio_read+0x42/0x90
[<ffffffff811917e4>] ? nameidata_to_filp+0x44/0x60
[<ffffffff81192b5e>] do_sync_read+0xfe/0x140
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811a6afa>] ? vfs_ioctl+0x2a/0xa0
[<ffffffff811a712e>] ? do_vfs_ioctl+0x7e/0x570
[<ffffffff8151c56d>] ? mutex_lock+0x1d/0x50
[<ffffffff8119333e>] vfs_read+0x9e/0x190
[<ffffffff8119347a>] sys_read+0x4a/0x90
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task suphp:581378 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
suphp D ffff88014b75e180 0 581378 580871 115 0x00000000
ffff880de5c2f938 0000000000000082 0000000000000000 ffff880c32833900
ffff880de5c2f8f8 ffffffff8140b5ec 0000000000000050 ffff88000002de40
0000000000000002 000000016573a0db ffff880de5c2ffd8 ffff880de5c2ffd8
Call Trace:
[<ffffffff8140b5ec>] ? dm_table_unplug_all+0x5c/0x100
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151b563>] io_schedule+0x73/0xc0
[<ffffffff811c77a0>] sync_buffer+0x40/0x50
[<ffffffff8151c31b>] __wait_on_bit_lock+0x5b/0xc0
[<ffffffff811c7760>] ? sync_buffer+0x0/0x50
[<ffffffff8151c3fc>] out_of_line_wait_on_bit_lock+0x7c/0x90
[<ffffffff81095340>] ? wake_bit_function+0x0/0x50
[<ffffffff811c8136>] __lock_buffer+0x36/0x40
[<ffffffff811c87a5>] __sync_dirty_buffer+0x35/0xe0
[<ffffffff811c8863>] sync_dirty_buffer+0x13/0x20
[<ffffffffa006f97a>] journal_dirty_data+0x19a/0x230 [jbd]
[<ffffffffa009adc0>] ext3_journal_dirty_data+0x20/0x50 [ext3]
[<ffffffffa009ae15>] journal_dirty_data_fn+0x25/0x30 [ext3]
[<ffffffffa0099a6d>] walk_page_buffers+0x8d/0xd0 [ext3]
[<ffffffffa009adf0>] ? journal_dirty_data_fn+0x0/0x30 [ext3]
[<ffffffffa009e2bb>] ext3_ordered_write_end+0xab/0x1c0 [ext3]
[<ffffffff81120f45>] generic_file_buffered_write_iter+0x175/0x2a0
[<ffffffff811213e8>] __generic_file_write_iter+0x188/0x380
[<ffffffff81121663>] __generic_file_aio_write+0x83/0xa0
[<ffffffff81121708>] generic_file_aio_write+0x88/0x100
[<ffffffff81192a1e>] do_sync_write+0xfe/0x140
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811931b1>] vfs_write+0xa1/0x190
[<ffffffff8119350a>] sys_write+0x4a/0x90
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
INFO: task suphp:581379 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
suphp D ffff88014b786200 0 581379 580835 115 0x00000000
ffff880437d35cd8 0000000000000086 0000000000000000 0000000000000098
0000000000000000 ffffea000bc4a5d0 0000020000000000 ffff8802f12976d0
00007fea148da0e0 000000016573a216 ffff880437d35fd8 ffff880437d35fd8
Call Trace:
[<ffffffff8151cb62>] __mutex_lock_slowpath+0x122/0x170
[<ffffffff8151c57a>] mutex_lock+0x2a/0x50
[<ffffffff811216f1>] generic_file_aio_write+0x71/0x100
[<ffffffff81192a1e>] do_sync_write+0xfe/0x140
[<ffffffff81095300>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811931b1>] vfs_write+0xa1/0x190
[<ffffffff8119350a>] sys_write+0x4a/0x90
[<ffffffff8100b182>] system_call_fastpath+0x16/0x1b
VERSION :
pve-manager: 3.0-23 (pve-manager/3.0/957f0862)
running kernel: 2.6.32-20-pve
proxmox-ve-2.6.32: 3.0-100
pve-kernel-2.6.32-20-pve: 2.6.32-100
pve-kernel-2.6.32-19-pve: 2.6.32-96
pve-kernel-2.6.32-18-pve: 2.6.32-88
lvm2: 2.02.95-pve3
clvm: 2.02.95-pve3
corosync-pve: 1.4.5-1
openais-pve: 1.1.4-3
libqb0: 0.11.1-2
redhat-cluster-pve: 3.2.0-2
resource-agents-pve: 3.9.2-4
fence-agents-pve: 4.0.0-1s
pve-cluster: 3.0-4
qemu-server: 3.0-20
pve-firmware: 1.0-22
libpve-common-perl: 3.0-4
libpve-access-control: 3.0-4
libpve-storage-perl: 3.0-8
vncterm: 1.1-4
vzctl: 4.0-1pve3
vzprocps: 2.0.11-2
vzquota: 3.1-2
pve-qemu-kvm: 1.4-13
ksm-control-daemon: 1.1-1
