Improving my current setup.

[Assign]

I currently have a cluster of 3 nodes.
Each node has:
- 2x AMD Epyc 7543 32-cores
- 2 TB of ram
- 2x 512GB Micron 7450 nvme in ZFS mirror for the os
- 8 7.68TB Micron 7450 for Ceph OSD's

The cluster has 2x 10Gbit interfaces on LACP over 2 seperate switches for redundancy and throughput for the VM's.
2 x 100Gbit interfaces in Active Passive over 2 switches for Ceph public and private.

OSD's are currently on ~26% used

I want to squeeze out some more write performance in ceph. What tweakes do I need to apply?

# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54

# devices
device 0 osd.0 class ssd
device 1 osd.1 class ssd
device 2 osd.2 class ssd
device 3 osd.3 class ssd
device 4 osd.4 class ssd
device 5 osd.5 class ssd
device 6 osd.6 class ssd
device 7 osd.7 class ssd
device 8 osd.8 class ssd
device 9 osd.9 class ssd
device 10 osd.10 class ssd
device 11 osd.11 class ssd
device 12 osd.12 class ssd
device 13 osd.13 class ssd
device 14 osd.14 class ssd
device 15 osd.15 class ssd
device 16 osd.16 class ssd
device 17 osd.17 class ssd
device 18 osd.18 class ssd
device 19 osd.19 class ssd
device 20 osd.20 class ssd
device 21 osd.21 class ssd
device 22 osd.22 class ssd
device 23 osd.23 class ssd

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 zone
type 10 region
type 11 root

# buckets
host pve1 {
    id -3        # do not change unnecessarily
    id -4 class ssd        # do not change unnecessarily
    # weight 55.89038
    alg straw2
    hash 0    # rjenkins1
    item osd.0 weight 6.98630
    item osd.1 weight 6.98630
    item osd.2 weight 6.98630
    item osd.3 weight 6.98630
    item osd.12 weight 6.98630
    item osd.13 weight 6.98630
    item osd.14 weight 6.98630
    item osd.15 weight 6.98630
}
host pve2 {
    id -5        # do not change unnecessarily
    id -6 class ssd        # do not change unnecessarily
    # weight 55.89038
    alg straw2
    hash 0    # rjenkins1
    item osd.4 weight 6.98630
    item osd.5 weight 6.98630
    item osd.6 weight 6.98630
    item osd.7 weight 6.98630
    item osd.16 weight 6.98630
    item osd.17 weight 6.98630
    item osd.18 weight 6.98630
    item osd.19 weight 6.98630
}
host pve3 {
    id -7        # do not change unnecessarily
    id -8 class ssd        # do not change unnecessarily
    # weight 55.89038
    alg straw2
    hash 0    # rjenkins1
    item osd.8 weight 6.98630
    item osd.9 weight 6.98630
    item osd.10 weight 6.98630
    item osd.11 weight 6.98630
    item osd.20 weight 6.98630
    item osd.21 weight 6.98630
    item osd.22 weight 6.98630
    item osd.23 weight 6.98630
}
root default {
    id -1        # do not change unnecessarily
    id -2 class ssd        # do not change unnecessarily
    # weight 167.67114
    alg straw2
    hash 0    # rjenkins1
    item pve1 weight 55.89038
    item pve2 weight 55.89038
    item pve3 weight 55.89038
}

# rules
rule replicated_rule {
    id 0
    type replicated
    step take default
    step chooseleaf firstn 0 type host
    step emit
}

# end crush map

[global]
    auth_client_required = cephx
    auth_cluster_required = cephx
    auth_service_required = cephx
    cluster_network = 172.17.1.21/24
    fsid = b9ff8e69-77a0-4d77-8e35-3153467062af
    mon_allow_pool_delete = true
    mon_host = 172.17.0.22 172.17.2.22 172.17.0.21 172.17.2.21 172.17.0.23 172.17.2.23
    ms_bind_ipv4 = true
    ms_bind_ipv6 = false
    osd_pool_default_min_size = 2
    osd_pool_default_size = 3
    public_network = 172.17.2.0/24 172.17.0.0/24

[client]
    keyring = /etc/pve/priv/$cluster.$name.keyring

[client.crash]
    keyring = /etc/pve/ceph/$cluster.$name.keyring

[mds]
    keyring = /var/lib/ceph/mds/ceph-$id/keyring

[mds.pve1]
    host = pve1
    mds_standby_for_name = pve

[mds.pve2]
    host = pve2
    mds_standby_for_name = pve

[mds.pve3]
    host = pve3
    mds_standby_for_name = pve

[mon.pve1]
    public_addr = 172.17.0.21

[mon.pve2]
    public_addr = 172.17.0.22

[mon.pve3]
    public_addr = 172.17.0.23