osd.83-master-gist

root@ceph-node201:~# ls -lah /var/lib/ceph/f4495398-a8c4-4ad9-8219-80c48625abdf/osd.83/
total 68K
drwx------  2 167 167 4.0K Jun 13 14:43 .
drwx------ 11 167 167 4.0K Jun 12 23:34 ..
lrwxrwxrwx  1 167 167   93 Jun 13 14:43 block -> /dev/ceph-ff60b68a-26fe-4294-8bec-4a9c329e858d/osd-block-73ab12e6-7758-4ebe-9319-5935309fcacd
lrwxrwxrwx  1 167 167   90 Jun 13 14:43 block.db -> /dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be
-rw-------  1 167 167   37 Jun 13 14:43 ceph_fsid
-rw-------  1 167 167  292 Jun 12 20:41 config
-rw-------  1 167 167   37 Jun 13 14:43 fsid
-rw-------  1 167 167   56 Jun 13 14:43 keyring
-rw-------  1 167 167    6 Jun 13 14:43 ready
-rw-------  1 167 167    3 Mar 16 18:37 require_osd_release
-rw-------  1 167 167   10 Jun 13 14:43 type
-rw-------  1 167 167   38 Jun 12 20:41 unit.configured
-rw-------  1 167 167   48 Mar 16 18:37 unit.created
-rw-------  1 167 167   28 Jun 12 20:41 unit.image
-rw-------  1 167 167  951 Jun 12 20:41 unit.poststop
-rw-------  1 167 167 2.1K Jun 12 20:41 unit.run
-rw-------  1 167 167    3 Jun 13 14:43 whoami
root@ceph-node201:~# ceph-bluestore-tool --log-level 30 show-label --dev /dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be
unable to read label for /dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be: (2) No such file or directory
root@ceph-node201:~# ceph-bluestore-tool --log-level 30 show-label --dev /dev/ceph-ff60b68a-26fe-4294-8bec-4a9c329e858d/osd-block-73ab12e6-7758-4ebe-9319-5935309fcacd
{
    "/dev/ceph-ff60b68a-26fe-4294-8bec-4a9c329e858d/osd-block-73ab12e6-7758-4ebe-9319-5935309fcacd": {
        "osd_uuid": "73ab12e6-7758-4ebe-9319-5935309fcacd",
        "size": 4000783007744,
        "btime": "2021-03-16T18:31:13.273236+0000",
        "description": "main",
        "bfm_blocks": "61047168",
        "bfm_blocks_per_key": "128",
        "bfm_bytes_per_block": "65536",
        "bfm_size": "4000783007744",
        "bluefs": "1",
        "ceph_fsid": "f4495398-a8c4-4ad9-8219-80c48625abdf",
        "kv_backend": "rocksdb",
        "magic": "ceph osd volume v026",
        "mkfs_done": "yes",
        "osd_key": "AQBt+VBg9AhhMRAAAt1iBhB3EPkqBzt5d4jqow==",
        "osdspec_affinity": "osd-spec-2xx",
        "ready": "ready",
        "require_osd_release": "14",
        "whoami": "83"
    }
}
root@ceph-node201:~# lvs -o lv_tags /dev/ceph-ff60b68a-26fe-4294-8bec-4a9c329e858d/osd-block-73ab12e6-7758-4ebe-9319-5935309fcacd
  LV Tags
  ceph.block_device=/dev/ceph-ff60b68a-26fe-4294-8bec-4a9c329e858d/osd-block-73ab12e6-7758-4ebe-9319-5935309fcacd,ceph.block_uuid=nbRXYl-fRrQ-qyYP-D93c-IGct-yKg4-rujDOX,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=f4495398-a8c4-4ad9-8219-80c48625abdf,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.db_device=/dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be,ceph.db_uuid=K09p3L-QV06-LOLO-uVeT-2ulz-GD3O-CEyRcs,ceph.encrypted=0,ceph.osd_fsid=73ab12e6-7758-4ebe-9319-5935309fcacd,ceph.osd_id=83,ceph.osdspec_affinity=osd-spec-2xx,ceph.type=block,ceph.vdo=0
root@ceph-node201:~# lvs -o lv_tags /dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be
  LV Tags
  ceph.block_device=/dev/ceph-ff60b68a-26fe-4294-8bec-4a9c329e858d/osd-block-73ab12e6-7758-4ebe-9319-5935309fcacd,ceph.block_uuid=nbRXYl-fRrQ-qyYP-D93c-IGct-yKg4-rujDOX,ceph.cephx_lockbox_secret=,ceph.cluster_fsid=f4495398-a8c4-4ad9-8219-80c48625abdf,ceph.cluster_name=ceph,ceph.crush_device_class=None,ceph.db_device=/dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be,ceph.db_uuid=K09p3L-QV06-LOLO-uVeT-2ulz-GD3O-CEyRcs,ceph.encrypted=0,ceph.osd_fsid=73ab12e6-7758-4ebe-9319-5935309fcacd,ceph.osd_id=83,ceph.osdspec_affinity=osd-spec-2xx,ceph.type=db,ceph.vdo=0
root@ceph-node201:~#

@NightDog

Could you run:

/usr/bin/ceph-bluestore-tool show-label --log-level=30 --dev /dev/ceph-00ed472c-f900-4dc3-9ddc-0e2f3b6547e3/osd-db-bb0eaa16-a1e0-4985-b4bd-74799e5226be -l /var/log/ceph/ceph-volume.log

and check last lines at /var/log/ceph/ceph-volume.log?

I've faced with the same issue and sumbit a bug at https://tracker.ceph.com/issues/52095

In my case it was:
2021-08-08T05:17:45.303+0000 7ff7fc9c0240 10 bluestore(/dev/sdb) _read_bdev_label
2021-08-08T05:17:45.303+0000 7ff7fc9c0240 2 bluestore(/dev/sdb) _read_bdev_label unable to decode label at offset 102: buffer::malformed_input: void bluestore_bdev_label_t::decode(ceph::buffer::v15_2_0::list::const_iterator&) decode past end of struct encoding

Hi oneumyvakin,

Unfortunately I nuked the disk and put it back, restoring some of the lost data in the pool from backup.

Thanks for getting back to me, and sorry I was not able to provide any additional information. Will keep that command in mind if I run into the issue again!

Regards