RHEL 7.3: kernel BUG at block/blk-core.c:2672!

Solution Verified - Updated -

Issue

  • RHEL kernel crashes after failing a BUG_ON sanity check in blk_finish_request with console messages:
[44302806.442634] kernel BUG at block/blk-core.c:2672!             
[44302806.443207] invalid opcode: 0000 [#1] SMP 
[44302806.443788] Modules linked in: sctp_diag sctp unix_diag af_packet_diag netlink_diag ip6table_filter ip6_tables iptable_filter tcp_diag udp_diag inet_diag bonding vfat fat xfs libcrc32c dm_mirror dm_region_hash dm_log ipmi_ssif iTCO_wdt ipmi_devintf iTCO_vendor_support mxm_wmi intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd mgag200 i2c_algo_bit pcspkr ttm drm_kms_helper syscopyarea sysfillrect sb_edac sysimgblt fb_sys_fops edac_core drm cdc_ether usbnet mii ses enclosure lpc_ich i2c_i801 tg3 i2c_core ipmi_si ipmi_msghandler ixgbe ptp mei_me pps_core mdio mei dca shpchp fjes wmi acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl lockd binfmt_misc grace sunrpc
[44302806.448818]  ip_tables sg ext4 jbd2 mbcache dm_mod sd_mod crc_t10dif crct10dif_common usb_storage mpt3sas scsi_transport_sas raid_class megaraid_sas
[44302806.450840] CPU: 26 PID: 0 Comm: swapper/26 Tainted: G        W      ------------   3.10.0-514.10.2.el7.x86_64 #1
[44302806.451939] Hardware name: LENOVO System x3650 M5: -[8871AC1]-/01KN179, BIOS -[TCE134L-2.60]- 02/08/2018
[44302806.453082] task: ffff8801538cce70 ti: ffff8801538e4000 task.ti: ffff8801538e4000
[44302806.454244] RIP: 0010:[<ffffffff812ef230>]  [<ffffffff812ef230>] blk_finish_request+0xf0/0x100
[44302806.455457] RSP: 0018:ffff882000183d60  EFLAGS: 00010087
[44302806.456670] RAX: 0000000000000246 RBX: ffff8820a2e3b900 RCX: 0000000000000828
[44302806.457926] RDX: 0000000008280828 RSI: 0000000000000000 RDI: ffff8820a2e3b900
[44302806.459234] RBP: ffff882000183d70 R08: 0000000000000246 R09: ffffffff81e3e480
[44302806.460518] R10: 0000000000000400 R11: 0000000000000400 R12: 0000000000000000
[44302806.461824] R13: ffff8820a48079c0 R14: 0000000000000000 R15: ffff881ffdee63c0
[44302806.463135] FS:  0000000000000000(0000) GS:ffff882000180000(0000) knlGS:0000000000000000
[44302806.464482] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[44302806.465839] CR2: 00000007a93f9ff0 CR3: 00000000019ba000 CR4: 00000000003407e0
[44302806.467226] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[44302806.468624] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[44302806.470020] Stack:
[44302806.471421]  ffff8820a2e3b900 0000000000000246 ffff882000183db0 ffffffff8145b0a6
[44302806.472895]  0000000008000002 ffff8820a48079c0 0000000008000002 ffff881ffdee63c0
[44302806.474386]  0000000000000000 ffff8820a2e3b900 ffff882000183e10 ffffffff8145b3c6
[44302806.475893] Call Trace:
[44302806.477382]  <IRQ> 
[44302806.477400] 
[44302806.478900]  [<ffffffff8145b0a6>] scsi_end_request+0x116/0x1e0
[44302806.480441]  [<ffffffff8145b3c6>] scsi_io_completion+0x1f6/0x670
[44302806.481985]  [<ffffffff81450495>] scsi_finish_command+0xd5/0x130
[44302806.483550]  [<ffffffff8145a792>] scsi_softirq_done+0x132/0x160
[44302806.485115]  [<ffffffff812f5bb0>] blk_done_softirq+0x90/0xc0
[44302806.486628]  [<ffffffff8108f2cf>] __do_softirq+0xef/0x280
[44302806.488158]  [<ffffffff8169849c>] call_softirq+0x1c/0x30
[44302806.489681]  [<ffffffff8102d365>] do_softirq+0x65/0xa0
[44302806.491216]  [<ffffffff8108f665>] irq_exit+0x115/0x120
[44302806.492767]  [<ffffffff81699038>] do_IRQ+0x58/0xf0
[44302806.494320]  [<ffffffff8168e1ad>] common_interrupt+0x6d/0x6d
[44302806.495889]  <EOI> 
[44302806.495905] 
[44302806.497477]  [<ffffffff81514392>] ? cpuidle_enter_state+0x52/0xc0
[44302806.499077]  [<ffffffff815144d9>] cpuidle_idle_call+0xd9/0x210
[44302806.500707]  [<ffffffff810350ee>] arch_cpu_idle+0xe/0x30
[44302806.502307]  [<ffffffff810e7e65>] cpu_startup_entry+0x245/0x290
[44302806.503909]  [<ffffffff8104f07a>] start_secondary+0x1ba/0x230
[44302806.505487] Code: 89 de e8 e4 13 00 00 e9 4b ff ff ff 83 7b 50 01 0f 85 58 ff ff ff 48 8b 43 38 48 8d b8 90 01 00 00 e8 f5 df e9 ff e9 43 ff ff ff <0f> 0b 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 
[44302806.508882] RIP  [<ffffffff812ef230>] blk_finish_request+0xf0/0x100
[44302806.510491]  RSP <ffff882000183d60>

The kernel panic stack trace is:

crash> bt
PID: 0      TASK: ffff8801538cce70  CPU: 26  COMMAND: "swapper/26"
 #0 [ffff882000183a10] machine_kexec at ffffffff81059bdb
 #1 [ffff882000183a70] __crash_kexec at ffffffff81105382
 #2 [ffff882000183b40] crash_kexec at ffffffff81105470
 #3 [ffff882000183b58] oops_end at ffffffff8168f2c8
 #4 [ffff882000183b80] die at ffffffff8102e93b
 #5 [ffff882000183bb0] do_trap at ffffffff8168e980
 #6 [ffff882000183c00] do_invalid_op at ffffffff8102b144
 #7 [ffff882000183cb0] invalid_op at ffffffff8169829e
    [exception RIP: blk_finish_request+0xf0]
    RIP: ffffffff812ef230  RSP: ffff882000183d60  RFLAGS: 00010087
    RAX: 0000000000000246  RBX: ffff8820a2e3b900  RCX: 0000000000000828
    RDX: 0000000008280828  RSI: 0000000000000000  RDI: ffff8820a2e3b900
    RBP: ffff882000183d70   R8: 0000000000000246   R9: ffffffff81e3e480
    R10: 0000000000000400  R11: 0000000000000400  R12: 0000000000000000
    R13: ffff8820a48079c0  R14: 0000000000000000  R15: ffff881ffdee63c0
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #8 [ffff882000183d78] scsi_end_request at ffffffff8145b0a6
 #9 [ffff882000183db8] scsi_io_completion at ffffffff8145b3c6
#10 [ffff882000183e18] scsi_finish_command at ffffffff81450495
#11 [ffff882000183e48] scsi_softirq_done at ffffffff8145a792
#12 [ffff882000183e78] blk_done_softirq at ffffffff812f5bb0
#13 [ffff882000183eb8] __do_softirq at ffffffff8108f2cf
#14 [ffff882000183f28] call_softirq at ffffffff8169849c
#15 [ffff882000183f40] do_softirq at ffffffff8102d365
#16 [ffff882000183f60] irq_exit at ffffffff8108f665
#17 [ffff882000183f78] do_IRQ at ffffffff81699038
--- <IRQ stack> ---
#18 [ffff8801538e7da8] ret_from_intr at ffffffff8168e1ad
    [exception RIP: cpuidle_enter_state+0x52]
    RIP: ffffffff81514392  RSP: ffff8801538e7e50  RFLAGS: 00000202
    RAX: 009d645cfff0864f  RBX: 000000000000f8a0  RCX: 0000000000000018
    RDX: 0000000225c17d03  RSI: ffff8801538e7fd8  RDI: 009d645cfff0864f
    RBP: ffff8801538e7e78   R8: 00000000000082b0   R9: 0000000000000018
    R10: 0000000000001730  R11: 0000000000000000  R12: ffff8801538e7e20
    R13: ffff88200018f8e0  R14: 0000000000000082  R15: ffff88200018f8e0
    ORIG_RAX: ffffffffffffff26  CS: 0010  SS: 0018
#19 [ffff8801538e7e80] cpuidle_idle_call at ffffffff815144d9
#20 [ffff8801538e7ec0] arch_cpu_idle at ffffffff810350ee
#21 [ffff8801538e7ed0] cpu_startup_entry at ffffffff810e7e65
#22 [ffff8801538e7f28] start_secondary at ffffffff8104f07a

Environment

  • RHEL 7
  • seen on kernel 3.10.0-514.10.2.el7
  • seen on 3.10.0-514.26.2.el7
  • SCSI storage accessed via a host bus adapter controlled by mpt3sas driver
  • virtio_scsi

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In

New to Red Hat?

Learn more about Red Hat subscriptions

Using a Red Hat product through a public cloud?

How to access this content