Ceph Compression

[paradox55]

I've set compression up on the pool but can't find any documentation on how to change the bluestore values on the OSD themselves to enable compression. How would one change this?

 

[AlexLup]

The pool itself is what decides which OSDs to reside on and also the compression. No need to instruct the OSDs individually.

 

[paradox55]

The pool itself is what decides which OSDs to reside on and also the compression. No need to instruct the OSDs individually.

That's not what I've read in the ceph documentations or the user mailing list. From what I understand both the pool and the OSD need to have compression enabled. It's apparently a bluestore value that has to be set.

However, none of the documentation covers how to change the bluestore value on the OSD. It only covers how to set it globally before OSD creation.

 

[paradox55]

Can I please get some assistance with this?

Some OSD's have compression. Others don't. Example: My 1TB HDD is compressing, but my 3TB one is not. See below output with bluestore_compressed.

root@e3-02:~# ceph daemon osd.6 perf dump | grep bluestore
    "bluestore": {
        "bluestore_allocated": 805973655552,
        "bluestore_stored": 796057956378,
        "bluestore_compressed": 0,
        "bluestore_compressed_allocated": 0,
        "bluestore_compressed_original": 0,
        "bluestore_onodes": 268200,
        "bluestore_onode_hits": 3681360,
        "bluestore_onode_misses": 1011262,
        "bluestore_onode_shard_hits": 3225497,
        "bluestore_onode_shard_misses": 146061,
        "bluestore_extents": 887695,
        "bluestore_blobs": 888189,
        "bluestore_buffers": 1280,
        "bluestore_buffer_bytes": 670572544,
        "bluestore_buffer_hit_bytes": 5858647801,
        "bluestore_buffer_miss_bytes": 267521275841,
        "bluestore_write_big": 323489,
        "bluestore_write_big_bytes": 1025906245632,
        "bluestore_write_big_blobs": 2012761,
        "bluestore_write_small": 896148,
        "bluestore_write_small_bytes": 6720568540,
        "bluestore_write_small_unused": 29253,
        "bluestore_write_small_deferred": 834797,
        "bluestore_write_small_pre_read": 629597,
        "bluestore_write_small_new": 2073779,
        "bluestore_txc": 1461400,
        "bluestore_onode_reshard": 260524,
        "bluestore_blob_split": 656,
        "bluestore_extent_compress": 661310,
        "bluestore_gc_merged": 0,
        "bluestore_read_eio": 0,
        "bluestore_reads_with_retries": 0,
        "bluestore_fragmentation_micros": 8,
        "bluestore_alloc_bytes": 6118712,
        "bluestore_alloc_items": 764839,
        "bluestore_cache_data_bytes": 670711808,
        "bluestore_cache_data_items": 1299,
        "bluestore_cache_onode_bytes": 175939200,
        "bluestore_cache_onode_items": 268200,
        "bluestore_cache_other_bytes": 897747679,
        "bluestore_cache_other_items": 21689067,
        "bluestore_fsck_bytes": 0,
        "bluestore_fsck_items": 0,
        "bluestore_txc_bytes": 3760,
        "bluestore_txc_items": 5,
        "bluestore_writing_deferred_bytes": 4493008,
        "bluestore_writing_deferred_items": 75,
        "bluestore_writing_bytes": 0,
        "bluestore_writing_items": 0,
    "throttle-bluestore_throttle_bytes": {
    "throttle-bluestore_throttle_deferred_bytes": {
root@e3-02:~# ceph daemon osd.5 perf dump | grep bluestore
    "bluestore": {
        "bluestore_allocated": 252878192640,
        "bluestore_stored": 263676029817,
        "bluestore_compressed": 15599780722,
        "bluestore_compressed_allocated": 17678532608,
        "bluestore_compressed_original": 31785549824,
        "bluestore_onodes": 144373,
        "bluestore_onode_hits": 1364973,
        "bluestore_onode_misses": 359117,
        "bluestore_onode_shard_hits": 2071869,
        "bluestore_onode_shard_misses": 10198,
        "bluestore_extents": 533419,
        "bluestore_blobs": 533352,
        "bluestore_buffers": 1794,
        "bluestore_buffer_bytes": 932683776,
        "bluestore_buffer_hit_bytes": 4612252151,
        "bluestore_buffer_miss_bytes": 139565332846,
        "bluestore_write_big": 125239,
        "bluestore_write_big_bytes": 408319492096,
        "bluestore_write_big_blobs": 798539,
        "bluestore_write_small": 503624,
        "bluestore_write_small_bytes": 3363297490,
        "bluestore_write_small_unused": 6494,
        "bluestore_write_small_deferred": 482356,
        "bluestore_write_small_pre_read": 414006,
        "bluestore_write_small_new": 819679,
        "bluestore_txc": 467793,
        "bluestore_onode_reshard": 86603,
        "bluestore_blob_split": 239,
        "bluestore_extent_compress": 690757,
        "bluestore_gc_merged": 0,
        "bluestore_read_eio": 0,
        "bluestore_reads_with_retries": 0,
        "bluestore_fragmentation_micros": 0,
        "bluestore_alloc_bytes": 2056024,
        "bluestore_alloc_items": 257003,
        "bluestore_cache_data_bytes": 932749312,
        "bluestore_cache_data_items": 1806,
        "bluestore_cache_onode_bytes": 94708688,
        "bluestore_cache_onode_items": 144373,
        "bluestore_cache_other_bytes": 496205344,
        "bluestore_cache_other_items": 11737685,
        "bluestore_fsck_bytes": 0,
        "bluestore_fsck_items": 0,
        "bluestore_txc_bytes": 37600,
        "bluestore_txc_items": 50,
        "bluestore_writing_deferred_bytes": 6772362,
        "bluestore_writing_deferred_items": 138,
        "bluestore_writing_bytes": 0,
        "bluestore_writing_items": 0,
    "throttle-bluestore_throttle_bytes": {
    "throttle-bluestore_throttle_deferred_bytes": {
root@e3-02:~#

 

[paradox55]

Bumpity bumpity bump.

If compression is really controlled on the pool side then I shouldn't be seeing different osd with different compression statistics. I'm only using two pools, both connected to cephfs.