Thanks, i noted that the 4th node wasn't started monitor. After remove and recriate this works. The folder /var/lib/ceph/osd/ceph-$id wasn't created. When i recreate the folder it's ok.
After this i forced start and now all it's ok.
# pveceph status
{
"quorum_names" : [
"2",
"0",
"1",
"3"
],
"fsid" : "901bdd67-0f28-4050-a0c9-68c45ee19dc1",
"quorum" : [
0,
1,
2,
3
],
"health" : {
"detail" : [
"mon.3 addr 192.168.xxx.x5:6789/0 clock skew 5.10404s > max 0.05s (latency 0.263264s)"
],
"health" : {
"health_services" : [
{
"mons" : [
{
"last_updated" : "2016-05-02 17:31:52.899567",
"name" : "2",
"kb_total" : 3546848,
"kb_used" : 2115984,
"kb_avail" : 1230980,
"health" : "HEALTH_OK",
"store_stats" : {
"bytes_total" : 30793980,
"bytes_log" : 2980886,
"bytes_sst" : 0,
"bytes_misc" : 27813094,
"last_updated" : "0.000000"
},
"avail_percent" : 34
},
{
"health" : "HEALTH_WARN",
"kb_avail" : 782408,
"kb_used" : 2564556,
"avail_percent" : 22,
"store_stats" : {
"last_updated" : "0.000000",
"bytes_misc" : 34191260,
"bytes_log" : 3143570,
"bytes_total" : 37334830,
"bytes_sst" : 0
},
"kb_total" : 3546848,
"health_detail" : "low disk space",
"last_updated" : "2016-05-02 17:31:23.090856",
"name" : "0"
},
{
"health_detail" : "low disk space",
"kb_total" : 3546848,
"name" : "1",
"last_updated" : "2016-05-02 17:31:27.150741",
"kb_avail" : 1009388,
"health" : "HEALTH_WARN",
"kb_used" : 2337576,
"avail_percent" : 28,
"store_stats" : {
"bytes_sst" : 0,
"bytes_total" : 35662769,
"bytes_log" : 3369694,
"bytes_misc" : 32293075,
"last_updated" : "0.000000"
}
},
{
"kb_used" : 2071456,
"health" : "HEALTH_OK",
"kb_avail" : 91831792,
"store_stats" : {
"last_updated" : "0.000000",
"bytes_misc" : 25529912,
"bytes_sst" : 0,
"bytes_log" : 3422445,
"bytes_total" : 28952357
},
"avail_percent" : 92,
"last_updated" : "2016-05-02 17:31:23.489128",
"name" : "3",
"kb_total" : 98952796
}
]
}
]
},
"overall_status" : "HEALTH_WARN",
"summary" : [
{
"severity" : "HEALTH_WARN",
"summary" : "mon.0 low disk space"
},
{
"summary" : "mon.1 low disk space",
"severity" : "HEALTH_WARN"
},
{
"summary" : "Monitor clock skew detected ",
"severity" : "HEALTH_WARN"
}
],
"timechecks" : {
"mons" : [
{
"skew" : 0,
"latency" : 0,
"name" : "2",
"health" : "HEALTH_OK"
},
{
"skew" : 0.000372,
"name" : "0",
"latency" : 0.263403,
"health" : "HEALTH_OK"
},
{
"health" : "HEALTH_OK",
"skew" : 0.00036,
"latency" : 0.263221,
"name" : "1"
},
{
"health" : "HEALTH_WARN",
"name" : "3",
"latency" : 0.263264,
"details" : "clock skew 5.10404s > max 0.05s",
"skew" : -5.104037
}
],
"epoch" : 2602,
"round" : 4,
"round_status" : "finished"
}
},
"monmap" : {
"fsid" : "901bdd67-0f28-4050-a0c9-68c45ee19dc1",
"modified" : "2016-05-02 16:59:29.970959",
"mons" : [
{
"rank" : 0,
"addr" : "192.168.xxx.x0:6789/0",
"name" : "2"
},
{
"rank" : 1,
"name" : "0",
"addr" : "192.168.xxx.x1:6789/0"
},
{
"name" : "1",
"addr" : "192.168.xxx.x2:6789/0",
"rank" : 2
},
{
"addr" : "192.168.xxx.x5:6789/0",
"name" : "3",
"rank" : 3
}
],
"epoch" : 6,
"created" : "2016-04-18 17:37:54.143234"
},
"mdsmap" : {
"by_rank" : [],
"up" : 0,
"in" : 0,
"max" : 0,
"epoch" : 1
},
"osdmap" : {
"osdmap" : {
"num_remapped_pgs" : 0,
"num_osds" : 7,
"epoch" : 1361,
"full" : false,
"num_in_osds" : 6,
"nearfull" : false,
"num_up_osds" : 6
}
},
"pgmap" : {
"version" : 399841,
"read_bytes_sec" : 16389,
"pgs_by_state" : [
{
"state_name" : "active+clean",
"count" : 64
}
],
"write_bytes_sec" : 126863,
"bytes_used" : 529655365632,
"bytes_total" : 4464561246208,
"op_per_sec" : 27,
"data_bytes" : 176576141414,
"bytes_avail" : 3934905880576,
"num_pgs" : 64
},
"election_epoch" : 2602
}
