RHCS 4 - ceph-osd assert with "/builddir/build/BUILD/ceph-14.2.22/src/os/bluestore/KernelDevice.cc: 1106: FAILED ceph_assert(off < size)"

Issue

Ceph OSDs randomly crashing with " FAILED ceph_assert(off < size)":

    -4> 2023-05-10 13:25:41.295 7fd3d62f3700  4 rocksdb: EVENT_LOG_v1 {"time_micros": 1683717941295291, "job": 7650, "event": "flush_started", "num_memtables": 1, "num_entries": 626917, "num_deletes": 120854, "total_data_size": 248345821, "memory_usage": 260050304, "flush_reason": "Write Buffer Full"}
    -3> 2023-05-10 13:25:41.295 7fd3d62f3700  4 rocksdb: [db/flush_job.cc:351] [default] [JOB 7650] Level-0 flush table #121512: started
    -2> 2023-05-10 13:25:41.636 7fd3d42ef700  5 prioritycache tune_memory target: 4294967296 mapped: 4324048896 unmapped: 1059995648 heap: 5384044544 old mem: 2620462370 new mem: 2603741011
    -1> 2023-05-10 13:25:41.676 7fd3d62f3700 -1 /builddir/build/BUILD/ceph-14.2.22/src/os/bluestore/KernelDevice.cc: In function 'virtual int KernelDevice::read_random(uint64_t, uint64_t, char*, bool)' thread 7fd3d62f3700 time 2023-05-10 13:25:41.647570
/builddir/build/BUILD/ceph-14.2.22/src/os/bluestore/KernelDevice.cc: 1106: FAILED ceph_assert(off < size)

 ceph version 14.2.22-128.el7cp (40a2bf9c4e79e39754d69a95cd51bd60991284be) nautilus (stable)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x14a) [0x55bb29655675]
 2: (()+0x4dd83d) [0x55bb2965583d]
 3: (KernelDevice::read_random(unsigned long, unsigned long, char*, bool)+0xeea) [0x55bb29cc20aa]
 4: (BlueFS::_read_random(BlueFS::FileReader*, unsigned long, unsigned long, char*)+0x56e) [0x55bb29c6b47e]
 5: (BlueRocksRandomAccessFile::Read(unsigned long, unsigned long, rocksdb::Slice*, char*) const+0x20) [0x55bb29c98cc0]
 6: (rocksdb::RandomAccessFileReader::Read(unsigned long, unsigned long, rocksdb::Slice*, char*) const+0x836) [0x55bb2a27ef06]
 7: (rocksdb::BlockFetcher::ReadBlockContents()+0x519) [0x55bb2a247b19]
 8: (rocksdb::BlockBasedTable::ReadCompressionDictBlock(rocksdb::BlockBasedTable::Rep*, rocksdb::FilePrefetchBuffer*, std::unique_ptr<rocksdb::BlockContents const, std::default_delete<rocksdb::BlockContents const> >*)+0x2b7) [0x55bb2a234387]
 9: (rocksdb::BlockBasedTable::GetUncompressionDict(rocksdb::BlockBasedTable::Rep*, rocksdb::FilePrefetchBuffer*, bool, rocksdb::GetContext*)+0x175) [0x55bb2a237bb5]
 10: (rocksdb::DataBlockIter* rocksdb::BlockBasedTable::NewDataBlockIterator<rocksdb::DataBlockIter>(rocksdb::BlockBasedTable::Rep*, rocksdb::ReadOptions const&, rocksdb::BlockHandle const&, rocksdb::DataBlockIter*, bool, bool, bool, rocksdb::GetContext*, rocksdb::Status, rocksdb::FilePrefetchBuffer*)+0x154) [0x55bb2a243e04]
 11: (rocksdb::BlockBasedTable::ReadRangeDelBlock(rocksdb::BlockBasedTable::Rep*, rocksdb::FilePrefetchBuffer*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, rocksdb::InternalKeyComparator const&)+0x16d) [0x55bb2a237fed]
 12: (rocksdb::BlockBasedTable::Open(rocksdb::ImmutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::BlockBasedTableOptions const&, rocksdb::InternalKeyComparator const&, std::unique_ptr<rocksdb::RandomAccessFileReader, std::default_delete<rocksdb::RandomAccessFileReader> >&&, unsigned long, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, rocksdb::SliceTransform const*, bool, bool, int, bool, unsigned long, rocksdb::TailPrefetchStats*)+0x966) [0x55bb2a239c46]
 13: (rocksdb::BlockBasedTableFactory::NewTableReader(rocksdb::TableReaderOptions const&, std::unique_ptr<rocksdb::RandomAccessFileReader, std::default_delete<rocksdb::RandomAccessFileReader> >&&, unsigned long, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, bool) const+0x60) [0x55bb2a22d0f0]
 14: (rocksdb::TableCache::GetTableReader(rocksdb::EnvOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileDescriptor const&, bool, unsigned long, bool, rocksdb::HistogramImpl*, std::unique_ptr<rocksdb::TableReader, std::default_delete<rocksdb::TableReader> >*, rocksdb::SliceTransform const*, bool, int, bool, bool)+0x499) [0x55bb2a1bb2f9]
 15: (rocksdb::TableCache::FindTable(rocksdb::EnvOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileDescriptor const&, rocksdb::Cache::Handle**, rocksdb::SliceTransform const*, bool, bool, rocksdb::HistogramImpl*, bool, int, bool)+0x306) [0x55bb2a1bbb96]
 16: (rocksdb::TableCache::NewIterator(rocksdb::ReadOptions const&, rocksdb::EnvOptions const&, rocksdb::InternalKeyComparator const&, rocksdb::FileMetaData const&, rocksdb::RangeDelAggregator*, rocksdb::SliceTransform const*, rocksdb::TableReader**, rocksdb::HistogramImpl*, bool, rocksdb::Arena*, bool, int, rocksdb::InternalKey const*, rocksdb::InternalKey const*)+0x414) [0x55bb2a1bcdb4]
 17: (rocksdb::BuildTable(std::string const&, rocksdb::Env*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::EnvOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::string const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x25b2) [0x55bb2a2ac662]
 18: (rocksdb::FlushJob::WriteLevel0Table()+0xf01) [0x55bb2a17b2a1]
 19: (rocksdb::FlushJob::Run(rocksdb::LogsWithPrepTracker*, rocksdb::FileMetaData*)+0xd9) [0x55bb2a17d1c9]
 20: (rocksdb::DBImpl::FlushMemTableToOutputFile(rocksdb::ColumnFamilyData*, rocksdb::MutableCFOptions const&, bool*, rocksdb::JobContext*, rocksdb::SuperVersionContext*, std::vector<unsigned long, std::allocator<unsigned long> >&, unsigned long, rocksdb::SnapshotChecker*, rocksdb::LogBuffer*, rocksdb::Env::Priority)+0x304) [0x55bb2a12e004]
 21: (rocksdb::DBImpl::FlushMemTablesToOutputFiles(rocksdb::autovector<rocksdb::DBImpl::BGFlushArg, 8ul> const&, bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::Env::Priority)+0x20a) [0x55bb2a135b3a]
 22: (rocksdb::DBImpl::BackgroundFlush(bool*, rocksdb::JobContext*, rocksdb::LogBuffer*, rocksdb::FlushReason*, rocksdb::Env::Priority)+0xc4f) [0x55bb2a13694f]
 23: (rocksdb::DBImpl::BackgroundCallFlush(rocksdb::Env::Priority)+0xc7) [0x55bb2a137f57]
 24: (rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long)+0x25a) [0x55bb2a306e6a]
 25: (rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*)+0x5d) [0x55bb2a306ffd]
 26: (()+0x1214e9f) [0x55bb2a38ce9f]
 27: (()+0x7ea5) [0x7fd3e8493ea5]
 28: (clone()+0x6d) [0x7fd3e7356b0d]