Ceph: MDS crashes with "ceph-17.2.6/src/mds/Locker.cc: 2364: FAILED ceph_assert(!cap->is_new())"

Issue

MDS crashes with "ceph-17.2.6/src/mds/Locker.cc: 2364: FAILED ceph_assert(!cap->is_new())"

The Ceph MDS crashes with the above assert, the full backtrace is seen below:

8:45.866120Z_d25f64bd-9471-41da-89a3-1ff63bc44ec1.txt
drwxrwxrwx+ 2 yank zeek   27 Oct 11 02:04 sosreport-20240830-121742
-bash 5.1 $ cat 0030-ceph.crash.info.mds.root.host6.phmvjs.txt 
{
    "assert_condition": "!cap->is_new()",
    "assert_file": "/builddir/build/BUILD/ceph-17.2.6/src/mds/Locker.cc",
    "assert_func": "int Locker::issue_caps(CInode*, Capability*)",
    "assert_line": 2364,
    "assert_msg": "/builddir/build/BUILD/ceph-17.2.6/src/mds/Locker.cc: In function 'int Locker::issue_caps(CInode*, Capability*)' thread 7fafb7d3c640 time 2024-08-30T09:07:02.727393+0000\n/builddir/build/BUILD/ceph-17.2.6/src/mds/Locker.cc: 2364: FAILED ceph_assert(!cap->is_new())\n",
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "/lib64/libc.so.6(+0x54db0) [0x7fafbcbcadb0]",
        "/lib64/libc.so.6(+0xa154c) [0x7fafbcc1754c]",
        "raise()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x188) [0x7fafbd226b4b]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x142caf) [0x7fafbd226caf]",
        "(Locker::issue_caps(CInode*, Capability*)+0x1904) [0x5583de8c7eb4]",
        "(Locker::simple_lock(SimpleLock*, bool*)+0x55d) [0x5583de8dd6fd]",
        "(Locker::xlock_start(SimpleLock*, boost::intrusive_ptr<MDRequestImpl>&)+0x2e9) [0x5583de8c2ba9]",
        "(Locker::acquire_locks(boost::intrusive_ptr<MDRequestImpl>&, MutationImpl::LockOpVec&, CInode*, bool)+0x20f5) [0x5583de8b8dd5]",
        "(Server::handle_client_setattr(boost::intrusive_ptr<MDRequestImpl>&)+0x1cd) [0x5583de7b739d]",
        "(Server::handle_client_request(boost::intrusive_ptr<MClientRequest const> const&)+0x615) [0x5583de79e665]",
        "(Server::dispatch(boost::intrusive_ptr<Message const> const&)+0x13c) [0x5583de78c18c]",
        "(MDSRank::_dispatch(boost::intrusive_ptr<Message const> const&, bool)+0x4f3) [0x5583de74a6b3]",
        "/usr/bin/ceph-mds(+0x12d7c9) [0x5583de72a7c9]",
        "(MDSContext::complete(int)+0x5c) [0x5583de9b4dac]",
        "/usr/bin/ceph-mds(+0x134abd) [0x5583de731abd]",
        "(MDSRankDispatcher::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&, MDSMap const&)+0x5b1) [0x5583de758f41]",
        "(MDSDaemon::handle_mds_map(boost::intrusive_ptr<MMDSMap const> const&)+0x1498) [0x5583de733138]",
        "(MDSDaemon::handle_core_message(boost::intrusive_ptr<Message const> const&)+0x367) [0x5583de7340e7]",
        "(MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x152) [0x5583de734832]",
        "(DispatchQueue::entry()+0x53a) [0x7fafbd41338a]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x3baff1) [0x7fafbd49eff1]",
        "/lib64/libc.so.6(+0x9f802) [0x7fafbcc15802]",
        "clone()"
    ],
    "ceph_version": "17.2.6-209.el9cp",
    "crash_id": "2024-08-30T09:07:02.729344Z_c22f396f-239f-4651-ba81-7f588037f44a",
    "entity_name": "mds.root.lwtxe04ypapd1i.phmvjs",
    "os_id": "rhel",
    "os_name": "Red Hat Enterprise Linux",
    "os_version": "9.3 (Plow)",
    "os_version_id": "9.3",
    "process_name": "ceph-mds",
    "stack_sig": "6ffdda65dbcb40a21fcadce01d3061d12d364e87239555307096e1f2bc880949",
    "timestamp": "2024-08-30T09:07:02.729344Z",
    "utsname_hostname": "lwtxe04ypapd1i",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-553.8.1.el8_10.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Fri Jun 14 03:19:37 EDT 2024"
}