8.2.4 - stucked pg x.x not deep-scrubbed since xxx

[kellogs]

The following has been shown for a long time

HEALTH_WARN: 16 pgs not deep-scrubbed in time
pg 8.14 not deep-scrubbed since 2024-09-27T19:42:47.463766+0700
pg 7.1b not deep-scrubbed since 2024-09-30T21:54:16.584697+0700
pg 8.1b not deep-scrubbed since 2024-09-29T23:22:37.812579+0700
pg 8.1d not deep-scrubbed since 2024-10-01T03:51:01.714780+0700
pg 6.13 not deep-scrubbed since 2024-09-26T07:16:31.387575+0700
pg 6.12 not deep-scrubbed since 2024-09-27T04:55:34.699188+0700
pg 8.3 not deep-scrubbed since 2024-09-29T20:57:40.973759+0700
pg 7.c not deep-scrubbed since 2024-10-01T22:57:28.955052+0700
pg 8.b not deep-scrubbed since 2024-10-01T09:15:55.223268+0700
pg 7.3 not deep-scrubbed since 2024-10-02T05:12:50.743544+0700
pg 7.18 not deep-scrubbed since 2024-09-30T17:57:34.550318+0700
pg 8.a not deep-scrubbed since 2024-09-30T10:37:06.429820+0700
pg 8.5 not deep-scrubbed since 2024-09-25T19:23:57.422280+0700
pg 8.9 not deep-scrubbed since 2024-09-30T17:57:33.493494+0700
pg 8.4 not deep-scrubbed since 2024-09-30T21:54:13.122287+0700
pg 8.10 not deep-scrubbed since 2024-10-02T20:07:42.793192+0700

root@stor-200-31:~# ceph pg stat
97 pgs: 7 active+clean+scrubbing+deep, 90 active+clean; 3.6 TiB data, 35 TiB used, 455 TiB / 491 TiB avail; 25 MiB/s rd, 32 MiB/s wr, 1.14k op/s

root@stor-200-31:~# ceph -s
cluster:
id: cc41de92-1edd-4f12-9803-884cdcea0c13
health: HEALTH_WARN
16 pgs not deep-scrubbed in time
16 pgs not scrubbed in time

services:
mon: 3 daemons, quorum stor-200-31,stor-200-30,stor-200-29 (age 4M)
mgr: stor-200-29(active, since 5M), standbys: stor-200-30, stor-200-31
mds: 1/1 daemons up, 2 standby
osd: 64 osds: 64 up (since 4w), 64 in (since 4w)

data:
volumes: 1/1 healthy
pools: 4 pools, 97 pgs
objects: 1.00M objects, 3.6 TiB
usage: 35 TiB used, 455 TiB / 491 TiB avail
pgs: 92 active+clean
5 active+clean+scrubbing+deep

io:
client: 30 MiB/s rd, 33 MiB/s wr, 78 op/s rd, 865 op/s wr

 

[kellogs]

example of ceph pg 8.14 query

root@stor-200-31:~# ceph pg 8.14 query
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "active+clean",
"epoch": 19002,
"up": [
13,
56,
34
],
"acting": [
13,
56,
34
],
"acting_recovery_backfill": [
"13",
"34",
"56"
],
"info": {
"pgid": "8.14",
"last_update": "19002'393000995",
"last_complete": "19002'393000995",
"log_tail": "19002'392990988",
"last_user_version": 393000995,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 1213,
"epoch_pool_created": 1213,
"last_epoch_started": 17525,
"last_interval_started": 17524,
"last_epoch_clean": 17525,
"last_interval_clean": 17524,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 17497,
"same_interval_since": 17524,
"same_primary_since": 15065,
"last_scrub": "15754'216006287",
"last_scrub_stamp": "2024-10-02T08:04:26.363903+0700",
"last_deep_scrub": "15500'202479289",
"last_deep_scrub_stamp": "2024-09-27T19:42:47.463766+0700",
"last_clean_scrub_stamp": "2024-10-02T08:04:26.363903+0700",
"prior_readable_until_ub": 0

"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2024-11-02T17:20:46.346536+0700",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
}
},
{
"name": "Started",
"enter_time": "2024-11-02T17:20:45.262359+0700"
}
],
"scrubber": {
"active": false,
"must_scrub": true,
"must_deep_scrub": false,
"must_repair": false,
"need_auto": false,
"scrub_reg_stamp": "1.000000",
"schedule": "queued for deep scrub"
},
"agent_state": {}
}
root@stor-200-31:~# ceph pg 8.14 query
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "active+clean",
"epoch": 19002,
"up": [
13,
56,
34
],
"acting": [
13,
56,
34
],
"acting_recovery_backfill": [
"13",
"34",
"56"
],

"scrubber": {
"active": false,
"must_scrub": true,
"must_deep_scrub": false,
"must_repair": false,
"need_auto": false,
"scrub_reg_stamp": "1.000000",
"schedule": "queued for deep scrub"
},
"agent_state": {}
}