RHEL 7.3: kernel BUG at block/blk-core.c:2672!

Solution Verified - Updated -

Issue

  • RHEL kernel crashes after failing a BUG_ON sanity check in blk_finish_request with console messages:
[44302806.442634] kernel BUG at block/blk-core.c:2672!             
[44302806.443207] invalid opcode: 0000 [#1] SMP 
[44302806.443788] Modules linked in: sctp_diag sctp unix_diag af_packet_diag netlink_diag ip6table_filter ip6_tables iptable_filter tcp_diag udp_diag inet_diag bonding vfat fat xfs libcrc32c dm_mirror dm_region_hash dm_log ipmi_ssif iTCO_wdt ipmi_devintf iTCO_vendor_support mxm_wmi intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd mgag200 i2c_algo_bit pcspkr ttm drm_kms_helper syscopyarea sysfillrect sb_edac sysimgblt fb_sys_fops edac_core drm cdc_ether usbnet mii ses enclosure lpc_ich i2c_i801 tg3 i2c_core ipmi_si ipmi_msghandler ixgbe ptp mei_me pps_core mdio mei dca shpchp fjes wmi acpi_pad acpi_power_meter nfsd auth_rpcgss nfs_acl lockd binfmt_misc grace sunrpc
[44302806.448818]  ip_tables sg ext4 jbd2 mbcache dm_mod sd_mod crc_t10dif crct10dif_common usb_storage mpt3sas scsi_transport_sas raid_class megaraid_sas
[44302806.450840] CPU: 26 PID: 0 Comm: swapper/26 Tainted: G        W      ------------   3.10.0-514.10.2.el7.x86_64 #1
[44302806.451939] Hardware name: LENOVO System x3650 M5: -[8871AC1]-/01KN179, BIOS -[TCE134L-2.60]- 02/08/2018
[44302806.453082] task: ffff8801538cce70 ti: ffff8801538e4000 task.ti: ffff8801538e4000
[44302806.454244] RIP: 0010:[<ffffffff812ef230>]  [<ffffffff812ef230>] blk_finish_request+0xf0/0x100
[44302806.455457] RSP: 0018:ffff882000183d60  EFLAGS: 00010087
[44302806.456670] RAX: 0000000000000246 RBX: ffff8820a2e3b900 RCX: 0000000000000828
[44302806.457926] RDX: 0000000008280828 RSI: 0000000000000000 RDI: ffff8820a2e3b900
[44302806.459234] RBP: ffff882000183d70 R08: 0000000000000246 R09: ffffffff81e3e480
[44302806.460518] R10: 0000000000000400 R11: 0000000000000400 R12: 0000000000000000
[44302806.461824] R13: ffff8820a48079c0 R14: 0000000000000000 R15: ffff881ffdee63c0
[44302806.463135] FS:  0000000000000000(0000) GS:ffff882000180000(0000) knlGS:0000000000000000
[44302806.464482] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[44302806.465839] CR2: 00000007a93f9ff0 CR3: 00000000019ba000 CR4: 00000000003407e0
[44302806.467226] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[44302806.468624] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[44302806.470020] Stack:
[44302806.471421]  ffff8820a2e3b900 0000000000000246 ffff882000183db0 ffffffff8145b0a6
[44302806.472895]  0000000008000002 ffff8820a48079c0 0000000008000002 ffff881ffdee63c0
[44302806.474386]  0000000000000000 ffff8820a2e3b900 ffff882000183e10 ffffffff8145b3c6
[44302806.475893] Call Trace:
[44302806.477382]  <IRQ> 
[44302806.477400] 
[44302806.478900]  [<ffffffff8145b0a6>] scsi_end_request+0x116/0x1e0
[44302806.480441]  [<ffffffff8145b3c6>] scsi_io_completion+0x1f6/0x670
[44302806.481985]  [<ffffffff81450495>] scsi_finish_command+0xd5/0x130
[44302806.483550]  [<ffffffff8145a792>] scsi_softirq_done+0x132/0x160
[44302806.485115]  [<ffffffff812f5bb0>] blk_done_softirq+0x90/0xc0
[44302806.486628]  [<ffffffff8108f2cf>] __do_softirq+0xef/0x280
[44302806.488158]  [<ffffffff8169849c>] call_softirq+0x1c/0x30
[44302806.489681]  [<ffffffff8102d365>] do_softirq+0x65/0xa0
[44302806.491216]  [<ffffffff8108f665>] irq_exit+0x115/0x120
[44302806.492767]  [<ffffffff81699038>] do_IRQ+0x58/0xf0
[44302806.494320]  [<ffffffff8168e1ad>] common_interrupt+0x6d/0x6d
[44302806.495889]  <EOI> 
[44302806.495905] 
[44302806.497477]  [<ffffffff81514392>] ? cpuidle_enter_state+0x52/0xc0
[44302806.499077]  [<ffffffff815144d9>] cpuidle_idle_call+0xd9/0x210
[44302806.500707]  [<ffffffff810350ee>] arch_cpu_idle+0xe/0x30
[44302806.502307]  [<ffffffff810e7e65>] cpu_startup_entry+0x245/0x290
[44302806.503909]  [<ffffffff8104f07a>] start_secondary+0x1ba/0x230
[44302806.505487] Code: 89 de e8 e4 13 00 00 e9 4b ff ff ff 83 7b 50 01 0f 85 58 ff ff ff 48 8b 43 38 48 8d b8 90 01 00 00 e8 f5 df e9 ff e9 43 ff ff ff <0f> 0b 0f 1f 40 00 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 
[44302806.508882] RIP  [<ffffffff812ef230>] blk_finish_request+0xf0/0x100
[44302806.510491]  RSP <ffff882000183d60>

The kernel panic stack trace is:

crash> bt
PID: 0      TASK: ffff8801538cce70  CPU: 26  COMMAND: "swapper/26"
 #0 [ffff882000183a10] machine_kexec at ffffffff81059bdb
 #1 [ffff882000183a70] __crash_kexec at ffffffff81105382
 #2 [ffff882000183b40] crash_kexec at ffffffff81105470
 #3 [ffff882000183b58] oops_end at ffffffff8168f2c8
 #4 [ffff882000183b80] die at ffffffff8102e93b
 #5 [ffff882000183bb0] do_trap at ffffffff8168e980
 #6 [ffff882000183c00] do_invalid_op at ffffffff8102b144
 #7 [ffff882000183cb0] invalid_op at ffffffff8169829e
    [exception RIP: blk_finish_request+0xf0]
    RIP: ffffffff812ef230  RSP: ffff882000183d60  RFLAGS: 00010087
    RAX: 0000000000000246  RBX: ffff8820a2e3b900  RCX: 0000000000000828
    RDX: 0000000008280828  RSI: 0000000000000000  RDI: ffff8820a2e3b900
    RBP: ffff882000183d70   R8: 0000000000000246   R9: ffffffff81e3e480
    R10: 0000000000000400  R11: 0000000000000400  R12: 0000000000000000
    R13: ffff8820a48079c0  R14: 0000000000000000  R15: ffff881ffdee63c0
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
 #8 [ffff882000183d78] scsi_end_request at ffffffff8145b0a6
 #9 [ffff882000183db8] scsi_io_completion at ffffffff8145b3c6
#10 [ffff882000183e18] scsi_finish_command at ffffffff81450495
#11 [ffff882000183e48] scsi_softirq_done at ffffffff8145a792
#12 [ffff882000183e78] blk_done_softirq at ffffffff812f5bb0
#13 [ffff882000183eb8] __do_softirq at ffffffff8108f2cf
#14 [ffff882000183f28] call_softirq at ffffffff8169849c
#15 [ffff882000183f40] do_softirq at ffffffff8102d365
#16 [ffff882000183f60] irq_exit at ffffffff8108f665
#17 [ffff882000183f78] do_IRQ at ffffffff81699038
--- <IRQ stack> ---
#18 [ffff8801538e7da8] ret_from_intr at ffffffff8168e1ad
    [exception RIP: cpuidle_enter_state+0x52]
    RIP: ffffffff81514392  RSP: ffff8801538e7e50  RFLAGS: 00000202
    RAX: 009d645cfff0864f  RBX: 000000000000f8a0  RCX: 0000000000000018
    RDX: 0000000225c17d03  RSI: ffff8801538e7fd8  RDI: 009d645cfff0864f
    RBP: ffff8801538e7e78   R8: 00000000000082b0   R9: 0000000000000018
    R10: 0000000000001730  R11: 0000000000000000  R12: ffff8801538e7e20
    R13: ffff88200018f8e0  R14: 0000000000000082  R15: ffff88200018f8e0
    ORIG_RAX: ffffffffffffff26  CS: 0010  SS: 0018
#19 [ffff8801538e7e80] cpuidle_idle_call at ffffffff815144d9
#20 [ffff8801538e7ec0] arch_cpu_idle at ffffffff810350ee
#21 [ffff8801538e7ed0] cpu_startup_entry at ffffffff810e7e65
#22 [ffff8801538e7f28] start_secondary at ffffffff8104f07a

Environment

  • RHEL 7
  • seen on kernel 3.10.0-514.10.2.el7
  • seen on 3.10.0-514.26.2.el7
  • SCSI storage accessed via a host bus adapter controlled by mpt3sas driver
  • virtio_scsi

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In