ceph - Probleme beim hinzufügen einer neuen osd

[Patrick987]

Hallo,

ich habe einen 3-node ceph cluster mit je 2 Disks. Jeden node möchte ich nun mit je einer weiteren HDD bestücken. Beim einbinden der HDD in ceph habe ich nun Probleme. Die osd.6 wird nicht in der Weboberfläche angezeigt. Die HDD steckt im node pvec1 welche ich im vorhinein mit "ceph-disk zap /dev/sdf" gelöscht habe. Was mache ich hierbei falsch? In der Crush-Map wird die zusätzliche OSD nicht angezeigt.

Vielen Dank im Voraus für eure Hilfe.

[EDIT] im englischen Forum: https://forum.proxmox.com/threads/problem-on-adding-new-osd.40741/

lg, Patrick

root@pvec1:~# pveceph createosd /dev/sdf
create OSD on /dev/sdf (xfs)
Caution: invalid backup GPT header, but valid main header; regenerating
backup header from main header.

****************************************************************************
Caution: Found protective or hybrid MBR and corrupt GPT. Using GPT, but disk
verification and recovery are STRONGLY recommended.
****************************************************************************
GPT data structures destroyed! You may now partition the disk using fdisk or
other utilities.
Creating new GPT entries.
The operation has completed successfully.
Setting name!
partNum is 1
REALLY setting name!
The operation has completed successfully.
Setting name!
partNum is 0
REALLY setting name!
The operation has completed successfully.
meta-data=/dev/sdf1              isize=2048   agcount=32, agsize=9115904 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=0        finobt=0
data     =                       bsize=4096   blocks=291708928, imaxpct=5
         =                       sunit=64     swidth=64 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=0
log      =internal log           bsize=4096   blocks=142464, version=2
         =                       sectsz=512   sunit=64 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0
Warning: The kernel is still using the old partition table.
The new table will be used at the next reboot.
The operation has completed successfully.

root@pvec1:~# ceph osd tree
ID WEIGHT  TYPE NAME      UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 4.88036 root default
-2 1.62679     host pvec1
 0 1.08620         osd.0       up  1.00000          1.00000
 1 0.54059         osd.1       up  1.00000          1.00000
-3 1.62679     host pvec2
 2 1.08620         osd.2       up  1.00000          1.00000
 3 0.54059         osd.3       up  1.00000          1.00000
-4 1.62679     host pvec3
 4 1.08620         osd.4       up  1.00000          1.00000
 5 0.54059         osd.5       up  1.00000          1.00000
 6       0 osd.6               up  1.00000          1.00000

root@pvec1:~# ceph tell osd.* version
osd.0: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}
osd.1: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}
osd.2: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}
osd.3: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}
osd.4: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}
osd.5: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}
osd.6: {
    "version": "ceph version 10.2.10 (5dc1e4c05cb68dbf62ae6fce3f0700e4654fdbbe)"
}

root@pvec1:~# pveversion -v
proxmox-ve: 4.4-104 (running kernel: 4.4.98-4-pve)
pve-manager: 4.4-21 (running version: 4.4-21/e0dadcf8)
pve-kernel-4.4.35-1-pve: 4.4.35-77
pve-kernel-4.4.35-2-pve: 4.4.35-79
pve-kernel-4.4.98-4-pve: 4.4.98-104
pve-kernel-4.4.67-1-pve: 4.4.67-92
pve-kernel-4.4.83-1-pve: 4.4.83-96
lvm2: 2.02.116-pve3
corosync-pve: 2.4.2-2~pve4+1
libqb0: 1.0.1-1
pve-cluster: 4.0-54
qemu-server: 4.0-114
pve-firmware: 1.1-11
libpve-common-perl: 4.0-96
libpve-access-control: 4.0-23
libpve-storage-perl: 4.0-76
pve-libspice-server1: 0.12.8-2
vncterm: 1.3-2
pve-docs: 4.4-4
pve-qemu-kvm: 2.9.1-6~pve4
pve-container: 1.0-104
pve-firewall: 2.0-33
pve-ha-manager: 1.0-41
ksm-control-daemon: 1.2-1
glusterfs-client: 3.5.2-2+deb8u3
lxc-pve: 2.0.7-4
lxcfs: 2.0.6-pve1
criu: 1.6.0-1
novnc-pve: 0.5-9
smartmontools: 6.5+svn4324-1~pve80
zfsutils: 0.6.5.9-pve15~bpo80
ceph: 10.2.10-1~bpo80+1

# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable straw_calc_version 1

# devices
device 0 osd.0
device 1 osd.1
device 2 osd.2
device 3 osd.3
device 4 osd.4
device 5 osd.5

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root

# buckets
host pvec1 {
        id -2           # do not change unnecessarily
        # weight 1.627
        alg straw
        hash 0  # rjenkins1
        item osd.0 weight 1.086
        item osd.1 weight 0.541
}
host pvec2 {
        id -3           # do not change unnecessarily
        # weight 1.627
        alg straw
        hash 0  # rjenkins1
        item osd.2 weight 1.086
        item osd.3 weight 0.541
}
host pvec3 {
        id -4           # do not change unnecessarily
        # weight 1.627
        alg straw
        hash 0  # rjenkins1
        item osd.4 weight 1.086
        item osd.5 weight 0.541
}
root default {
        id -1           # do not change unnecessarily
        # weight 4.880
        alg straw
        hash 0  # rjenkins1
        item pvec1 weight 1.627
        item pvec2 weight 1.627
        item pvec3 weight 1.627
}

# rules
rule replicated_ruleset {
        ruleset 0
        type replicated
        min_size 1
        max_size 10
        step take default
        step chooseleaf firstn 0 type host
        step emit
}

# end crush map