Hello.
The disks in our PROXMOX cluster are slow. Virtual machines placed not on SSD pool drive work with the disk slowly and spend a lot of time on disk I/O. Virtual machines placed not on HDD pool drive work with the disk wery slowly
Help fix this flaw
I built ceph with 3 node
We use SSD as Journal
We use a hp JH295A 16-Port 10-Gigabit Switch for communication between nodes.
MTU on network interfaces set to 9000
iperf shows 10 gigabit speed
ping -M do -s 8972 10.172.213.2 succeeds
Firmware and BIOS updated to latest wersion on motherboard and HBA adapter
We use these disks on all cluster nodes (All servers are exactly the same in hardware configuration).
The disks in our PROXMOX cluster are slow. Virtual machines placed not on SSD pool drive work with the disk slowly and spend a lot of time on disk I/O. Virtual machines placed not on HDD pool drive work with the disk wery slowly
Help fix this flaw
I built ceph with 3 node
We use SSD as Journal
We use a hp JH295A 16-Port 10-Gigabit Switch for communication between nodes.
MTU on network interfaces set to 9000
iperf shows 10 gigabit speed
ping -M do -s 8972 10.172.213.2 succeeds
Firmware and BIOS updated to latest wersion on motherboard and HBA adapter
We use these disks on all cluster nodes (All servers are exactly the same in hardware configuration).
Supermicro mb MBD-X10DRI-T-O (E-ATX;DP S2011 R3;16xDDR4;10xSATA3 RAID 0,1,5,10;6PCIe;2*10GbE;IPMI)
32 x Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz (2 Sockets)
RAM usage 30.98% (117.06 GiB of 377.80 GiB)
LSI SAS 9300-4i SGL HBA 12Gb/s (LSI00346) JBOD, 1024 Non-RAID SAS/SATA devices, 1x SFF8643
32 x Intel(R) Xeon(R) CPU E5-2620 v4 @ 2.10GHz (2 Sockets)
RAM usage 30.98% (117.06 GiB of 377.80 GiB)
LSI SAS 9300-4i SGL HBA 12Gb/s (LSI00346) JBOD, 1024 Non-RAID SAS/SATA devices, 1x SFF8643
[global]
auth client required = cephx
auth cluster required = cephx
auth service required = cephx
cluster network = 10.172.213.0/24
fsid = f58eb7fa-de46-426f-9e63-0b38004c495b
keyring = /etc/pve/priv/$cluster.$name.keyring
mon allow pool delete = true
osd journal size = 5120
osd pool default min size = 2
osd pool default size = 3
public network = 10.172.213.0/24
[osd]
keyring = /var/lib/ceph/osd/ceph-$id/keyring
osd scrub begin hour = 19
osd scrub end hour = 9
[mon.pve2]
host = pve2
mon addr = 10.172.213.2:6789
[mon.pve1]
host = pve1
mon addr = 10.172.213.1:6789
[mon.pve3]
host = pve3
mon addr = 10.172.213.3:6789
auth client required = cephx
auth cluster required = cephx
auth service required = cephx
cluster network = 10.172.213.0/24
fsid = f58eb7fa-de46-426f-9e63-0b38004c495b
keyring = /etc/pve/priv/$cluster.$name.keyring
mon allow pool delete = true
osd journal size = 5120
osd pool default min size = 2
osd pool default size = 3
public network = 10.172.213.0/24
[osd]
keyring = /var/lib/ceph/osd/ceph-$id/keyring
osd scrub begin hour = 19
osd scrub end hour = 9
[mon.pve2]
host = pve2
mon addr = 10.172.213.2:6789
[mon.pve1]
host = pve1
mon addr = 10.172.213.1:6789
[mon.pve3]
host = pve3
mon addr = 10.172.213.3:6789
root@pve1:~# pveversion -v
proxmox-ve: 5.4-2 (running kernel: 4.15.18-20-pve)
pve-manager: 5.4-13 (running version: 5.4-13/aee6f0ec)
pve-kernel-4.15: 5.4-8
pve-kernel-4.13: 5.2-2
pve-kernel-4.15.18-20-pve: 4.15.18-46
pve-kernel-4.15.18-15-pve: 4.15.18-40
pve-kernel-4.15.18-12-pve: 4.15.18-36
pve-kernel-4.15.18-11-pve: 4.15.18-34
pve-kernel-4.15.18-10-pve: 4.15.18-32
pve-kernel-4.15.18-9-pve: 4.15.18-30
pve-kernel-4.15.18-7-pve: 4.15.18-27
pve-kernel-4.15.18-4-pve: 4.15.18-23
pve-kernel-4.15.18-2-pve: 4.15.18-21
pve-kernel-4.15.18-1-pve: 4.15.18-19
pve-kernel-4.15.17-1-pve: 4.15.17-9
pve-kernel-4.13.16-4-pve: 4.13.16-51
pve-kernel-4.13.16-2-pve: 4.13.16-48
pve-kernel-4.13.16-1-pve: 4.13.16-46
pve-kernel-4.10.17-2-pve: 4.10.17-20
ceph: 12.2.12-pve1
corosync: 2.4.4-pve1
criu: 2.11.1-1~bpo90
glusterfs-client: 3.8.8-1
ksm-control-daemon: 1.2-2
libjs-extjs: 6.0.1-2
libpve-access-control: 5.1-12
libpve-apiclient-perl: 2.0-5
libpve-common-perl: 5.0-54
libpve-guest-common-perl: 2.0-20
libpve-http-server-perl: 2.0-14
libpve-storage-perl: 5.0-44
libqb0: 1.0.3-1~bpo9
lvm2: 2.02.168-pve6
lxc-pve: 3.1.0-6
lxcfs: 3.0.3-pve1
novnc-pve: 1.0.0-3
proxmox-widget-toolkit: 1.0-28
pve-cluster: 5.0-38
pve-container: 2.0-40
pve-docs: 5.4-2
pve-edk2-firmware: 1.20190312-1
pve-firewall: 3.0-22
pve-firmware: 2.0-7
pve-ha-manager: 2.0-9
pve-i18n: 1.1-4
pve-libspice-server1: 0.14.1-2
pve-qemu-kvm: 3.0.1-4
pve-xtermjs: 3.12.0-1
qemu-server: 5.0-54
smartmontools: 6.5+svn4324-1
spiceterm: 3.0-5
vncterm: 1.5-3
zfsutils-linux: 0.7.13-pve1~bpo2
root@pve1:~#
proxmox-ve: 5.4-2 (running kernel: 4.15.18-20-pve)
pve-manager: 5.4-13 (running version: 5.4-13/aee6f0ec)
pve-kernel-4.15: 5.4-8
pve-kernel-4.13: 5.2-2
pve-kernel-4.15.18-20-pve: 4.15.18-46
pve-kernel-4.15.18-15-pve: 4.15.18-40
pve-kernel-4.15.18-12-pve: 4.15.18-36
pve-kernel-4.15.18-11-pve: 4.15.18-34
pve-kernel-4.15.18-10-pve: 4.15.18-32
pve-kernel-4.15.18-9-pve: 4.15.18-30
pve-kernel-4.15.18-7-pve: 4.15.18-27
pve-kernel-4.15.18-4-pve: 4.15.18-23
pve-kernel-4.15.18-2-pve: 4.15.18-21
pve-kernel-4.15.18-1-pve: 4.15.18-19
pve-kernel-4.15.17-1-pve: 4.15.17-9
pve-kernel-4.13.16-4-pve: 4.13.16-51
pve-kernel-4.13.16-2-pve: 4.13.16-48
pve-kernel-4.13.16-1-pve: 4.13.16-46
pve-kernel-4.10.17-2-pve: 4.10.17-20
ceph: 12.2.12-pve1
corosync: 2.4.4-pve1
criu: 2.11.1-1~bpo90
glusterfs-client: 3.8.8-1
ksm-control-daemon: 1.2-2
libjs-extjs: 6.0.1-2
libpve-access-control: 5.1-12
libpve-apiclient-perl: 2.0-5
libpve-common-perl: 5.0-54
libpve-guest-common-perl: 2.0-20
libpve-http-server-perl: 2.0-14
libpve-storage-perl: 5.0-44
libqb0: 1.0.3-1~bpo9
lvm2: 2.02.168-pve6
lxc-pve: 3.1.0-6
lxcfs: 3.0.3-pve1
novnc-pve: 1.0.0-3
proxmox-widget-toolkit: 1.0-28
pve-cluster: 5.0-38
pve-container: 2.0-40
pve-docs: 5.4-2
pve-edk2-firmware: 1.20190312-1
pve-firewall: 3.0-22
pve-firmware: 2.0-7
pve-ha-manager: 2.0-9
pve-i18n: 1.1-4
pve-libspice-server1: 0.14.1-2
pve-qemu-kvm: 3.0.1-4
pve-xtermjs: 3.12.0-1
qemu-server: 5.0-54
smartmontools: 6.5+svn4324-1
spiceterm: 3.0-5
vncterm: 1.5-3
zfsutils-linux: 0.7.13-pve1~bpo2
root@pve1:~#
root@pve1:~# for i in /dev/sd?; do hdparm -W $i; done
/dev/sda:
write-caching = 0 (off)
/dev/sdb:
write-caching = 0 (off)
/dev/sdc:
write-caching = 0 (off)
/dev/sdd:
write-caching = 0 (off)
/dev/sde:
write-caching = 0 (off)
/dev/sdf:
write-caching = 0 (off)
/dev/sdg:
write-caching = 0 (off)
/dev/sdh:
write-caching = 0 (off)
/dev/sdi:
write-caching = 0 (off)
/dev/sdj:
write-caching = 0 (off)
/dev/sdk:
write-caching = 1 (on)
/dev/sdl:
write-caching = 1 (on)
root@pve1:~#
/dev/sda:
write-caching = 0 (off)
/dev/sdb:
write-caching = 0 (off)
/dev/sdc:
write-caching = 0 (off)
/dev/sdd:
write-caching = 0 (off)
/dev/sde:
write-caching = 0 (off)
/dev/sdf:
write-caching = 0 (off)
/dev/sdg:
write-caching = 0 (off)
/dev/sdh:
write-caching = 0 (off)
/dev/sdi:
write-caching = 0 (off)
/dev/sdj:
write-caching = 0 (off)
/dev/sdk:
write-caching = 1 (on)
/dev/sdl:
write-caching = 1 (on)
root@pve1:~#
root@pve1:~# cat /sys/block/*/queue/write_cache
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write back
root@pve1:~#
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write through
write back
root@pve1:~#
