Kernel panic occurs at __list_add or __list_del due to "unable to handle kernel NULL pointer dereference"

Solution Unverified - Updated -

Issue

  • Kernel panic occurs with the message BUG: unable to handle kernel NULL pointer dereference at 0000000000000008.
[950387.644619] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008
[950387.645350] IP: [<ffffffff969bd15f>] __list_add+0xf/0xc0
[950387.646045] PGD 0
[950387.646654] Oops: 0000 [#1] SMP
[950387.647266] Modules linked in: udp_diag unix_diag af_packet_diag netlink_diag tcp_diag inet_diag falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_14712(E) nfsv3 nfs_acl nfs lockd grace fscache team_mode_activebackup team dell_rbu rpcrdma sunrpc ib_isert iscsi_target_mod ib_iser libiscsi scsi_transport_iscsi ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ipoib rdma_ucm ext4 ib_uverbs ib_umad mbcache jbd2 rdma_cm ib_cm iw_cm skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel mgag200 i2c_algo_bit ttm drm_kms_helper kvm irqbypass syscopyarea sysfillrect dell_smbios dcdbas dell_wmi_descriptor sysimgblt fb_sys_fops drm crc32_pclmul ghash_clmulni_intel aesni_intel drm_panel_orientation_quirks lrw gf128mul glue_helper ablk_helper cryptd mei_me
[950387.651138]  pcspkr sg wdat_wdt mei bnxt_re ib_core i2c_i801 lpc_ich wmi ipmi_si ipmi_devintf ipmi_msghandler tpm_crb acpi_power_meter acpi_pad dm_multipath binfmt_misc ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic ahci crct10dif_pclmul libahci crct10dif_common bnxt_en crc32c_intel libata megaraid_sas devlink nfit libnvdimm dm_mirror dm_region_hash dm_log dm_mod
[950387.653673] CPU: 2 PID: 241564 Comm: TaniumClient Kdump: loaded Tainted: P            E  ------------   3.10.0-1160.88.1.el7.x86_64 #1
[950387.655018] Hardware name: XXXXX, BIOS 2.16.1 08/17/2022
[950387.655666] task: ffff8a142a6d9080 ti: ffff89cd599b0000 task.ti: ffff89cd599b0000
[950387.656316] RIP: 0010:[<ffffffff969bd15f>]  [<ffffffff969bd15f>] __list_add+0xf/0xc0
[950387.657010] RSP: 0000:ffff89cd599b3aa8  EFLAGS: 00010082
[950387.657717] RAX: 0000000000000000 RBX: 0000000000000000 RCX: ffffe6bf00bab8c0
[950387.658440] RDX: 0000000000000000 RSI: ffff89b5fffda048 RDI: ffffe6bf00bab8e0
[950387.659133] RBP: ffff89cd599b3ac0 R08: ffffe6bf00bab8e0 R09: 0000000000000000
[950387.659788] R10: ffff89b5fffbed00 R11: ffff896ef9369518 R12: 0000000000000000
[950387.660445] R13: ffff89b5fffda048 R14: 00000000ffffffff R15: ffff89b5fffd9fe0
[950387.661118] FS:  00007fa9a2f75780(0000) GS:ffff89b47f240000(0000) knlGS:0000000000000000
[950387.661757] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[950387.662383] CR2: 0000000000000008 CR3: 000000193a608000 CR4: 00000000007607e0
[950387.663000] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[950387.663603] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[950387.664182] PKRU: 55555554
[950387.664724] Call Trace:
[950387.665257]  [<ffffffff967d0e6e>] __rmqueue+0xee/0x460
[950387.665811]  [<ffffffff967d37cc>] get_page_from_freelist+0x53c/0xad0
[950387.666375]  [<ffffffff967d3ee4>] __alloc_pages_nodemask+0x184/0xbf0
[950387.666952]  [<ffffffff967cae81>] ? generic_file_aio_read+0x1d1/0x2a0
[950387.667493]  [<ffffffff96db7112>] ? down_read+0x12/0x40
[950387.668035]  [<ffffffff96829359>] alloc_pages_vma+0xa9/0x210
[950387.668627]  [<ffffffff9680227f>] handle_mm_fault+0xb9f/0x1190
[950387.669192]  [<ffffffff96dbf7d3>] __do_page_fault+0x213/0x510
[950387.669756]  [<ffffffff96dbfb05>] do_page_fault+0x35/0x90
[950387.670295]  [<ffffffff96dbb7b8>] page_fault+0x28/0x30
[950387.670805] Code: 63 e7 ff b8 f4 ff ff ff e9 33 ff ff ff b8 f4 ff ff ff e9 29 ff ff ff 0f 1f 44 00 00 55 48 89 e5 41 55 49 89 f5 41 54 49 89 d4 53 <4c> 8b 42 08 48 89 fb 49 39 f0 75 2e 4d 8b 45 00 4d 39 c4 75 6c
[950387.671948] RIP  [<ffffffff969bd15f>] __list_add+0xf/0xc0
[950387.672525]  RSP <ffff89cd599b3aa8>
[950387.673097] CR2: 0000000000000008
  • Another pattern.
[714084.693589] BUG: unable to handle kernel NULL pointer dereference at 0000000000000008
[714084.694155] IP: [<ffffffffb51a6571>] __list_del_entry+0x31/0xd0
[714084.694717] PGD 1426c35f067 PUD 14226624067 PMD 0 
[714084.695281] Oops: 0000 [#1] SMP 
[714084.695834] Modules linked in: tcp_diag udp_diag inet_diag unix_diag af_packet_diag netlink_diag rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfsv3 nfs_acl nfs lockd grace fscache falcon_lsm_serviceable(PE) falcon_nf_netcontain(PE) falcon_kal(E) falcon_lsm_pinned_14306(E) team_mode_activebackup team ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp ib_ipoib ib_umad rpcrdma sunrpc rdma_ucm ib_uverbs ib_iser rdma_cm iw_cm ib_cm libiscsi scsi_transport_iscsi skx_edac nfit libnvdimm vfat coretemp fat intel_rapl iosf_mbi dm_multipath mgag200 kvm i2c_algo_bit irqbypass crc32_pclmul ghash_clmulni_intel ttm aesni_intel lrw gf128mul drm_kms_helper glue_helper ablk_helper syscopyarea sysfillrect bnxt_re ses pcspkr cryptd enclosure sysimgblt ib_core fb_sys_fops sg drm mei_me joydev
[714084.699725]  mei ipmi_si lpc_ich hpilo hpwdt drm_panel_orientation_quirks wmi ipmi_devintf ipmi_msghandler tcp_htcp sch_fq_codel binfmt_misc ip_tables xfs libcrc32c sd_mod lpfc nvmet_fc nvmet crc_t10dif crct10dif_generic crct10dif_pclmul crc32c_intel nvme_fc bnxt_en nvme_fabrics smartpqi nvme_core scsi_transport_fc scsi_transport_sas scsi_tgt devlink crct10dif_common dm_mirror dm_region_hash dm_log dm_mod fuse
[714084.702668] CPU: 38 PID: 8615 Comm: TaniumCX Kdump: loaded Tainted: P    B   W   E  ------------   3.10.0-1160.76.1.el7.x86_64 #1
[714084.704217] Hardware name: XXXX, BIOS U34 01/23/2021
[714084.705026] task: ffff970bc4ade300 ti: ffff96fce2b58000 task.ti: ffff96fce2b58000
[714084.705843] RIP: 0010:[<ffffffffb51a6571>]  [<ffffffffb51a6571>] __list_del_entry+0x31/0xd0
[714084.706677] RSP: 0018:ffff96fce2b5b460  EFLAGS: 00010046
[714084.707507] RAX: ffff964cfffda1c8 RBX: ffffe0f0204da020 RCX: dead000000000200
[714084.708343] RDX: 0000000000000000 RSI: ffff964cfffda000 RDI: ffffe0f0204da020
[714084.709180] RBP: ffff96fce2b5b460 R08: ffffe0f0204da020 R09: 00000000024ae111
[714084.710010] R10: 0000000000028048 R11: ffff96dc3f373208 R12: 0000000000000001
[714084.710842] R13: ffff964cfffda1c8 R14: 0000000000000003 R15: ffffe0f0204da020
[714084.711679] FS:  00007fc950a00700(0000) GS:ffff964c3fc00000(0000) knlGS:0000000000000000
[714084.712527] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[714084.713371] CR2: 0000000000000008 CR3: 0000014f75df2000 CR4: 00000000007607e0
[714084.714223] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[714084.715070] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[714084.715907] PKRU: 55555554
[714084.716738] Call Trace:
[714084.717567]  [<ffffffffb51a661d>] list_del+0xd/0x30
[714084.718402]  [<ffffffffb4fc5f6a>] __rmqueue+0x8a/0x460
[714084.719234]  [<ffffffffb4fc89cc>] get_page_from_freelist+0x6cc/0xac0
[714084.720068]  [<ffffffffb4fc8f44>] __alloc_pages_nodemask+0x184/0xbe0
[714084.720900]  [<ffffffffb51a65c2>] ? __list_del_entry+0x82/0xd0
[714084.721729]  [<ffffffffb51a661d>] ? list_del+0xd/0x30
[714084.722553]  [<ffffffffb4fc5f6a>] ? __rmqueue+0x8a/0x460
[714084.723377]  [<ffffffffb50193b8>] alloc_pages_current+0x98/0x110
[714084.724211]  [<ffffffffb5027aa3>] new_slab+0x393/0x4e0
[714084.725032]  [<ffffffffb5027fbc>] ___slab_alloc+0x3cc/0x520
[714084.725851]  [<ffffffffb4fc1565>] ? mempool_alloc_slab+0x15/0x20
[714084.726669]  [<ffffffffb4fc1565>] ? mempool_alloc_slab+0x15/0x20
[714084.727478]  [<ffffffffb5027e19>] ? ___slab_alloc+0x229/0x520
[714084.728287]  [<ffffffffb4fc1565>] ? mempool_alloc_slab+0x15/0x20
[714084.729095]  [<ffffffffb5582ef5>] __slab_alloc+0x40/0x5c
[714084.729898]  [<ffffffffb502917b>] kmem_cache_alloc+0x19b/0x1f0
[714084.730695]  [<ffffffffb4fc1565>] mempool_alloc_slab+0x15/0x20
[714084.731484]  [<ffffffffb4fc16be>] mempool_alloc+0x6e/0x170
[714084.732264]  [<ffffffffb4fc16be>] ? mempool_alloc+0x6e/0x170
[714084.733041]  [<ffffffffb508c6d2>] bvec_alloc+0x92/0x120
[714084.733799]  [<ffffffffb508c973>] bio_alloc_bioset+0x213/0x310
[714084.734545]  [<ffffffffc031059b>] __clone_and_map_data_bio+0x19b/0x280 [dm_mod]
[714084.735278]  [<ffffffffc0310951>] __split_and_process_bio+0x2d1/0x520 [dm_mod]
[714084.735987]  [<ffffffffb5150000>] ? ftrace_raw_output_block_bio_merge+0x90/0x90
[714084.736685]  [<ffffffffc0310be3>] dm_make_request+0x43/0xa0 [dm_mod]
[714084.737359]  [<ffffffffb51559c7>] generic_make_request+0x147/0x380
[714084.738017]  [<ffffffffb518cade>] ? radix_tree_tag_clear+0x9e/0xe0
[714084.738652]  [<ffffffffb5155c70>] submit_bio+0x70/0x150
[714084.739267]  [<ffffffffb4fc9f24>] ? __test_set_page_writeback+0x54/0x1a0
[714084.739892]  [<ffffffffc0545f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[714084.720900]  [<ffffffffb51a65c2>] ? __list_del_entry+0x82/0xd0
[714084.721729]  [<ffffffffb51a661d>] ? list_del+0xd/0x30
[714084.722553]  [<ffffffffb4fc5f6a>] ? __rmqueue+0x8a/0x460
[714084.723377]  [<ffffffffb50193b8>] alloc_pages_current+0x98/0x110
[714084.724211]  [<ffffffffb5027aa3>] new_slab+0x393/0x4e0
[714084.725032]  [<ffffffffb5027fbc>] ___slab_alloc+0x3cc/0x520
[714084.725851]  [<ffffffffb4fc1565>] ? mempool_alloc_slab+0x15/0x20
[714084.726669]  [<ffffffffb4fc1565>] ? mempool_alloc_slab+0x15/0x20
[714084.727478]  [<ffffffffb5027e19>] ? ___slab_alloc+0x229/0x520
[714084.728287]  [<ffffffffb4fc1565>] ? mempool_alloc_slab+0x15/0x20
[714084.729095]  [<ffffffffb5582ef5>] __slab_alloc+0x40/0x5c
[714084.729898]  [<ffffffffb502917b>] kmem_cache_alloc+0x19b/0x1f0
[714084.730695]  [<ffffffffb4fc1565>] mempool_alloc_slab+0x15/0x20
[714084.731484]  [<ffffffffb4fc16be>] mempool_alloc+0x6e/0x170
[714084.732264]  [<ffffffffb4fc16be>] ? mempool_alloc+0x6e/0x170
[714084.733041]  [<ffffffffb508c6d2>] bvec_alloc+0x92/0x120
[714084.733799]  [<ffffffffb508c973>] bio_alloc_bioset+0x213/0x310
[714084.734545]  [<ffffffffc031059b>] __clone_and_map_data_bio+0x19b/0x280 [dm_mod]
[714084.735278]  [<ffffffffc0310951>] __split_and_process_bio+0x2d1/0x520 [dm_mod]
[714084.735987]  [<ffffffffb5150000>] ? ftrace_raw_output_block_bio_merge+0x90/0x90
[714084.736685]  [<ffffffffc0310be3>] dm_make_request+0x43/0xa0 [dm_mod]
[714084.737359]  [<ffffffffb51559c7>] generic_make_request+0x147/0x380
[714084.738017]  [<ffffffffb518cade>] ? radix_tree_tag_clear+0x9e/0xe0
[714084.738652]  [<ffffffffb5155c70>] submit_bio+0x70/0x150
[714084.739267]  [<ffffffffb4fc9f24>] ? __test_set_page_writeback+0x54/0x1a0
[714084.739892]  [<ffffffffc0545f31>] xfs_submit_ioend.isra.12+0x61/0xe0 [xfs]
[714084.740495]  [<ffffffffc0546724>] xfs_do_writepage+0x494/0x550 [xfs]
[714084.741065]  [<ffffffffb4fca99c>] write_cache_pages+0x21c/0x470
[714084.741633]  [<ffffffffc0546290>] ? xfs_vm_writepages+0xa0/0xa0 [xfs]
[714084.742194]  [<ffffffffc054625b>] xfs_vm_writepages+0x6b/0xa0 [xfs]
[714084.742734]  [<ffffffffb4fcb9f1>] do_writepages+0x21/0x50
[714084.743264]  [<ffffffffb4fbfe75>] __filemap_fdatawrite_range+0x65/0x80
[714084.743793]  [<ffffffffb4fbffc1>] filemap_write_and_wait_range+0x41/0x90
[714084.744334]  [<ffffffffc0550d66>] xfs_file_fsync+0x66/0x1c0 [xfs]
[714084.744858]  [<ffffffffb5084235>] do_fsync+0x55/0x80
[714084.745377]  [<ffffffffb5084503>] SyS_fdatasync+0x13/0x20
[714084.745893]  [<ffffffffb5599f92>] system_call_fastpath+0x25/0x2a
[714084.746399] Code: 00 01 00 00 00 00 ad de 48 8b 47 08 48 89 e5 48 39 ca 74 29 48 b9 00 02 00 00 00 00 ad de 48 39 c8 74 7a 4c 8b 00 4c 39 c7 75 53 <4c> 8b 42 08 4c 39 c7 75 2b 48 89 42 08 48 89 10 5d c3 49 89 d0 
[714084.747519] RIP  [<ffffffffb51a6571>] __list_del_entry+0x31/0xd0
[714084.748062]  RSP <ffff96fce2b5b460>
[714084.748602] CR2: 0000000000000008

Environment

  • Red Hat Enterprise Linux 7
    • 3.10.0-1160.88.1.el7.x86_64
    • 3.10.0-1160.76.1.el7.x86_64
  • Third party module: falcon_*

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In

New to Red Hat?

Learn more about Red Hat subscriptions

Using a Red Hat product through a public cloud?

How to access this content