RHEL-9.5: kernel panic at kfree from nfsd4_encode_fattr4 or kernel panic after WARNING a free_large_kmalloc

Solution Verified - Updated -

Issue

[   75.378676] BUG: unable to handle page fault for address: ffffcfbb6b000108
[   75.378687] #PF: supervisor read access in kernel mode
[   75.378691] #PF: error_code(0x0000) - not-present page
[   75.378694] PGD 0 P4D 0 
[   75.378700] Oops: 0000 [#1] PREEMPT SMP PTI
[   75.378706] CPU: 5 PID: 3039 Comm: nfsd Kdump: loaded Tainted: G S                -------  ---  5.14.0-503.14.1.el9_5.x86_64 #1
[   75.378712] Hardware name: HP ProLiant DL380 Gen9/ProLiant DL380 Gen9, BIOS P89 07/21/2019
[   75.378714] RIP: 0010:kfree+0x4b/0x120
[   75.378732] Code: 80 48 01 e8 0f 82 dd 00 00 00 48 c7 c2 00 00 00 80 48 2b 15 3f a1 38 01 48 01 d0 48 c1 e8 0c 48 c1 e0 06 48 03 05 1d a1 38 01 <48> 8b 50 08 48 89 c7 f6 c2 01 0f 85 a4 00 00 00 66 90 48 8b 07 f6
[   75.378735] RSP: 0018:ffff999fcb89bb40 EFLAGS: 00010286
[   75.378740] RAX: ffffcfbb6b000100 RBX: ffffffffba91e2e0 RCX: ffff88964a5d0f40
[   75.378743] RDX: 0000776a40000000 RSI: ffffffffb973c918 RDI: 0000000000004810
[   75.378746] RBP: 0000000000004810 R08: ffffffffbb007940 R09: ffff889d9f9744b0
[   75.378749] R10: 0000000000000326 R11: ffff889d9f971c64 R12: 0000000000004810
[   75.378751] R13: 0000000000000000 R14: ffff889df1b3be00 R15: ffff889df3d40000
[   75.378755] FS:  0000000000000000(0000) GS:ffff889d9f940000(0000) knlGS:0000000000000000
[   75.378759] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   75.378761] CR2: ffffcfbb6b000108 CR3: 00000009dda10004 CR4: 00000000001726f0
[   75.378765] Call Trace:
[   75.378768]  <TASK>
[   75.378772]  ? show_trace_log_lvl+0x1c4/0x2df
[   75.378785]  ? show_trace_log_lvl+0x1c4/0x2df
[   75.378792]  ? security_release_secctx+0x28/0x40
[   75.378802]  ? __die_body.cold+0x8/0xd
[   75.378808]  ? page_fault_oops+0x134/0x170
[   75.378817]  ? kernelmode_fixup_or_oops+0x84/0x110
[   75.378822]  ? exc_page_fault+0xa8/0x150
[   75.378835]  ? asm_exc_page_fault+0x22/0x30
[   75.378845]  ? security_release_secctx+0x28/0x40
[   75.378850]  ? kfree+0x4b/0x120
[   75.378857]  security_release_secctx+0x28/0x40
[   75.378865]  nfsd4_encode_fattr4+0x2cc/0x4f0 [nfsd]
[   75.379026]  ? __kmem_cache_alloc_node+0x18f/0x2e0
[   75.379038]  ? security_prepare_creds+0x71/0xa0
[   75.379046]  ? security_prepare_creds+0x71/0xa0
[   75.379050]  ? __kmalloc+0x4b/0x140
[   75.379055]  ? __pfx_bpf_lsm_cred_prepare+0x10/0x10
[   75.379061]  ? security_prepare_creds+0x47/0xa0
[   75.379065]  ? sysvec_reschedule_ipi+0x26/0x100
[   75.379073]  ? asm_sysvec_reschedule_ipi+0x16/0x20
[   75.379083]  nfsd4_encode_getattr+0x2b/0x40 [nfsd]
[   75.379186]  nfsd4_encode_operation+0xa6/0x2b0 [nfsd]
[   75.379288]  nfsd4_proc_compound+0x1d0/0x700 [nfsd]
[   75.379389]  nfsd_dispatch+0xe9/0x220 [nfsd]
[   75.379479]  svc_process_common+0x2e7/0x650 [sunrpc]
[   75.379649]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[   75.379741]  svc_process+0x12d/0x170 [sunrpc]
[   75.379868]  svc_handle_xprt+0x448/0x580 [sunrpc]
[   75.380003]  svc_recv+0x17a/0x2c0 [sunrpc]
[   75.380137]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[   75.380226]  nfsd+0x84/0xb0 [nfsd]
[   75.380312]  kthread+0xe0/0x100
[   75.380323]  ? __pfx_kthread+0x10/0x10
[   75.380330]  ret_from_fork+0x2c/0x50
[   75.380339]  </TASK>
[   75.380341] Modules linked in: rpcsec_gss_krb5 uinput snd_seq_dummy snd_hrtimer snd_seq snd_timer snd_seq_device snd soundcore vhost_net vhost vhost_iotlb tap tun nft_objref nf_conntrack_netbios_ns nf_conntrack_broadcast nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 bonding nf_defrag_ipv4 tls bridge stp llc rfkill ip_set nf_tables nfnetlink qrtr ext4 vfat fat mbcache jbd2 dm_multipath intel_rapl_msr intel_rapl_common sb_edac x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel ipmi_ssif kvm rapl iTCO_wdt iTCO_vendor_support intel_cstate acpi_ipmi joydev intel_uncore pcspkr ipmi_si i2c_i801 acpi_tad hpilo ioatdma lpc_ich i2c_smbus dca acpi_power_meter ipmi_devintf ipmi_msghandler nfsd nfs_acl lockd auth_rpcgss grace sunrpc xfs libcrc32c sr_mod cdrom sd_mod mgag200 t10_pi sg drm_kms_helper i2c_algo_bit ahci drm_shmem_helper libahci crct10dif_pclmul crc32_pclmul crc32c_intel libata drm ghash_clmulni_intel
[   75.380454]  tg3 hpsa hpwdt scsi_transport_sas wmi dm_mirror dm_region_hash dm_log dm_mod fuse
[   75.380468] CR2: ffffcfbb6b000108
  • Another pattern of log:
[ 1094.184106] NFSD: Using nfsdcld client tracking operations.
[ 1094.184110] NFSD: no clients to reclaim, skipping NFSv4 grace period (net f0000000)
[49763.501571] ------------[ cut here ]------------
[49763.501574] WARNING: CPU: 6 PID: 29948 at mm/slab_common.c:957 free_large_kmalloc+0x5a/0x80
[49763.501582] Modules linked in: rpcrdma rdma_cm iw_cm ib_cm ib_core nfsd nfsv3 nfs_acl rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache netfs tls rfkill vsock_loopback vmw_vsock_virtio_transport_common vmw_vsock_vmci_transport vsock sunrpc intel_rapl_msr intel_rapl_common intel_uncore_frequency_common nfit libnvdimm kvm_intel kvm vmw_balloon rapl pcspkr vmw_vmci i2c_piix4 joydev ext4 mbcache jbd2 vmwgfx sr_mod drm_ttm_helper cdrom ttm ata_generic drm_kms_helper sd_mod t10_pi crct10dif_pclmul sg ata_piix crc32_pclmul crc32c_intel drm libata ghash_clmulni_intel vmxnet3 vmw_pvscsi serio_raw fuse
[49763.501620] CPU: 6 PID: 29948 Comm: nfsd Kdump: loaded Not tainted 5.14.0-503.15.1.el9_5.x86_64 #1
[49763.501622] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[49763.501623] RIP: 0010:free_large_kmalloc+0x5a/0x80
[49763.501626] Code: da 9c 5b fa be 06 00 00 00 48 89 ef e8 af 25 0a 00 80 e7 02 74 01 fb 48 83 c4 08 44 89 e6 48 89 ef 5b 5d 41 5c e9 d6 28 04 00 <0f> 0b 45 31 e4 80 3d 13 0e fc 01 00 ba 00 f0 ff ff 0f 84 8b 9a 90
[49763.501627] RSP: 0018:ffffa4f1c0dd7b28 EFLAGS: 00010246
[49763.501629] RAX: 0017ffffe4020056 RBX: ffffffff8411e2e0 RCX: ffff8970d2b27108
[49763.501630] RDX: ffffd5278519d3c8 RSI: ffffffffc0e1047c RDI: ffffd52785148400
[49763.501630] RBP: ffffd52785148400 R08: ffffffff84806c80 R09: ffff89761dfb44b0
[49763.501631] R10: 0000000000000032 R11: ffff89761dfb1c64 R12: ffffffffc0e1047c
[49763.501631] R13: 0000000000000000 R14: ffff896f0c8d8900 R15: ffff896f78e2c000
[49763.501632] FS:  0000000000000000(0000) GS:ffff89761df80000(0000) knlGS:0000000000000000
[49763.501633] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[49763.501634] CR2: 00007f4a562d3000 CR3: 000000014134c005 CR4: 0000000000770ef0
[49763.501658] PKRU: 55555554
[49763.501659] Call Trace:
[49763.501660]  <TASK>
[49763.501663]  ? show_trace_log_lvl+0x1c4/0x2df
[49763.501671]  ? show_trace_log_lvl+0x1c4/0x2df
[49763.501673]  ? security_release_secctx+0x25/0x40
[49763.501676]  ? free_large_kmalloc+0x5a/0x80
[49763.501678]  ? __warn+0x7e/0xd0
[49763.501681]  ? free_large_kmalloc+0x5a/0x80
[49763.501683]  ? report_bug+0x100/0x140
[49763.501686]  ? handle_bug+0x3c/0x70
[49763.501689]  ? exc_invalid_op+0x14/0x70
[49763.501690]  ? asm_exc_invalid_op+0x16/0x20
[49763.501694]  ? _fh_update.part.0.isra.0+0x4c/0x90 [nfsd]
[49763.501739]  ? _fh_update.part.0.isra.0+0x4c/0x90 [nfsd]
[49763.501773]  ? free_large_kmalloc+0x5a/0x80
[49763.501777]  ? _fh_update.part.0.isra.0+0x4c/0x90 [nfsd]
[49763.501821]  security_release_secctx+0x25/0x40
[49763.501823]  nfsd4_encode_fattr4+0x2cc/0x4f0 [nfsd]
[49763.501854]  ? __kmem_cache_alloc_node+0x18f/0x2e0
[49763.501858]  ? sort_r+0x22b/0x2b0
[49763.501860]  ? sort+0x2d/0x50
[49763.501862]  ? __pfx_gid_cmp+0x10/0x10
[49763.501865]  ? nfsd_setuser+0x110/0x270 [nfsd]
[49763.501891]  ? nfsd_setuser_and_check_port+0x4a/0xc0 [nfsd]
[49763.501914]  ? _fh_update.part.0.isra.0+0x4c/0x90 [nfsd]
[49763.501937]  nfsd4_encode_getattr+0x2b/0x40 [nfsd]
[49763.501964]  nfsd4_encode_operation+0xa3/0x2b0 [nfsd]
[49763.501992]  nfsd4_proc_compound+0x1d0/0x700 [nfsd]
[49763.502020]  nfsd_dispatch+0xe6/0x220 [nfsd]
[49763.502044]  svc_process_common+0x2e4/0x650 [sunrpc]
[49763.502101]  ? __pfx_nfsd_dispatch+0x10/0x10 [nfsd]
[49763.502124]  svc_process+0x12d/0x170 [sunrpc]
[49763.502158]  svc_handle_xprt+0x448/0x580 [sunrpc]
[49763.502195]  svc_recv+0x17a/0x2c0 [sunrpc]
[49763.502229]  ? __pfx_nfsd+0x10/0x10 [nfsd]
[49763.502253]  nfsd+0x84/0xb0 [nfsd]
[49763.502276]  kthread+0xdd/0x100
[49763.502279]  ? __pfx_kthread+0x10/0x10
[49763.502282]  ret_from_fork+0x29/0x50
[49763.502285]  </TASK>
[49763.502286] ---[ end trace 0000000000000000 ]---
[49763.502287] object pointer: 0x0000000046133336
[51594.056513] list_del corruption. prev->next should be ffffd527851483c8, but was ffffd52785140808
[51594.056544] ------------[ cut here ]------------
[51594.056545] kernel BUG at lib/list_debug.c:51!
[51594.056556] invalid opcode: 0000 [#1] PREEMPT SMP NOPTI
[51594.056558] CPU: 4 PID: 80 Comm: kcompactd0 Kdump: loaded Tainted: G        W         -------  ---  5.14.0-503.15.1.el9_5.x86_64 #1
[51594.056560] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[51594.056560] RIP: 0010:__list_del_entry_valid.cold+0x31/0x47
[51594.056566] Code: b3 07 84 e8 46 78 fe ff 0f 0b 48 c7 c7 98 b4 07 84 e8 38 78 fe ff 0f 0b 48 89 f2 48 89 fe 48 c7 c7 58 b4 07 84 e8 24 78 fe ff <0f> 0b 48 89 fe 4c 89 c2 48 c7 c7 20 b4 07 84 e8 10 78 fe ff 0f 0b

Environment

  • Red Hat Enterprise Linux 9.5.z
  • Seen on kernel-5.14.0-503.14.1.el9_5, kernel-5.14.0-503.15.1.el9_5
  • [nfsd]

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In

New to Red Hat?

Learn more about Red Hat subscriptions

Using a Red Hat product through a public cloud?

How to access this content