A hard lockup occurs due to a spinlock deadlock between start_flush_work() and __queue_work(). The spinlock is the system_wq's per-CPU pwq->pool->lock spinlock.

Issue

• A hard lockup occurs due to a spinlock deadlock between start_flush_work() and __queue_work(). The spinlock is the system_wq's per-CPU pwq->pool->lock spinlock.

[ 1226.656176] Kernel panic - not syncing: Hard LOCKUP
[ 1226.656177] CPU: 2 PID: 162603 Comm: mmfsd Kdump: loaded Tainted: G        W  OE    --------- -  - 4.18.0-305.34.2.el8_4.x86_64 #1
[ 1226.656177] Hardware name: HPE ProLiant DL380 Gen10 Plus/ProLiant DL380 Gen10 Plus, BIOS U46 09/22/2021
[ 1226.656177] Call Trace:
[ 1226.656177]  <NMI>
[ 1226.656177]  dump_stack+0x5c/0x80
[ 1226.656178]  panic+0xe7/0x2a9
[ 1226.656178]  nmi_panic.cold.9+0xc/0xc
[ 1226.656178]  watchdog_overflow_callback.cold.7+0x5c/0x70
[ 1226.656178]  __perf_event_overflow+0x52/0xf0
[ 1226.656178]  handle_pmi_common+0x204/0x2a0
[ 1226.656178]  intel_pmu_handle_irq+0xbf/0x160
[ 1226.656179]  perf_event_nmi_handler+0x2d/0x50
[ 1226.656179]  nmi_handle+0x63/0x110
[ 1226.656179]  default_do_nmi+0x49/0x100
[ 1226.656179]  do_nmi+0x17e/0x1e0
[ 1226.656179]  end_repeat_nmi+0x16/0x6f
[ 1226.656180] RIP: 0010:native_queued_spin_lock_slowpath+0x59/0x1d0
[ 1226.656180] Code: ff 75 6d f0 0f ba 2f 08 0f 92 c0 0f b6 c0 c1 e0 08 89 c2 8b 07 30 e4 09 d0 a9 00 01 ff ff 75 47 85 c0 74 0e 8b 07 84 c0 74 08 <f3> 90 8b 07 84 c0 75 f8 b8 01 00 00 00 66 89 07 c3 8b 37 81 fe 00
[ 1226.656180] RSP: 0018:ff6ba9887fcdb780 EFLAGS: 00000002
[ 1226.656181] RAX: 0000000001240101 RBX: ff3ba27f3faaf700 RCX: 0000000000000000
[ 1226.656181] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ff3ba27f3faa9840
[ 1226.656181] RBP: ff3ba27bbec2d8d0 R08: ff3ba201c78056b0 R09: ff3ba201c78056d8
[ 1226.656181] R10: 0000000000000000 R11: ffffffff90c58308 R12: ff3ba201c7c21000
[ 1226.656182] R13: 0000000000002000 R14: 0000000000000002 R15: ffffffff9077b620
[ 1226.656182]  ? native_queued_spin_lock_slowpath+0x59/0x1d0
[ 1226.656182]  ? native_queued_spin_lock_slowpath+0x59/0x1d0
[ 1226.656182]  </NMI>
[ 1226.656182]  _raw_spin_lock+0x1c/0x20
[ 1226.656183]  __queue_work+0x221/0x3f0
[ 1226.656183]  queue_work_on+0x34/0x40
[ 1226.656183]  soft_cursor+0x194/0x220
[ 1226.656183]  bit_cursor+0x3d2/0x610
[ 1226.656183]  ? bit_putcs+0x550/0x550
[ 1226.656183]  ? fbcon_cursor+0xff/0x170
[ 1226.656184]  hide_cursor+0x2a/0xa0
[ 1226.656184]  vt_console_print+0x3bd/0x400
[ 1226.656184]  console_unlock+0x379/0x4c0
[ 1226.656184]  vprintk_emit+0x184/0x290
[ 1226.656185]  ? __switch_to_asm+0x41/0x70
[ 1226.656185]  printk+0x58/0x6f
[ 1226.656185]  ? __switch_to_asm+0x41/0x70
[ 1226.656185]  ? disk_check_events+0x150/0x150
[ 1226.656185]  __warn_printk+0x46/0x87
[ 1226.656186]  ? disk_check_events+0x150/0x150
[ 1226.656186]  ? disk_check_events+0x150/0x150
[ 1226.656186]  check_flush_dependency+0x9d/0x130
[ 1226.656186]  flush_work+0x8b/0x1d0
[ 1226.656186]  ? try_to_del_timer_sync+0x4d/0x80
[ 1226.656186]  ? del_timer_sync+0x25/0x40
[ 1226.656187]  ? schedule_timeout+0x19b/0x2f0
[ 1226.656187]  ? work_busy+0x80/0x80
[ 1226.656187]  __cancel_work_timer+0x105/0x190
[ 1226.656187]  ? exact_lock+0xd/0x20
[ 1226.656187]  ? kobj_lookup+0xf1/0x160
[ 1226.656187]  disk_block_events+0x6f/0x90
[ 1226.656188]  __blkdev_get+0x70/0x510
[ 1226.656188]  blkdev_get+0xaf/0x180
[ 1226.656188]  ? bd_acquire+0xac/0xd0
[ 1226.656188]  ? bd_acquire+0xd0/0xd0
[ 1226.656188]  do_dentry_open+0x132/0x340
[ 1226.656188]  path_openat+0x53e/0x14f0
[ 1226.656189]  ? entry_SYSCALL_64_after_hwframe+0xb8/0xca
[ 1226.656189]  ? __switch_to_asm+0x35/0x70
[ 1226.656189]  ? __switch_to_asm+0x35/0x70
[ 1226.656189]  ? __switch_to_asm+0x41/0x70
[ 1226.656189]  do_filp_open+0x93/0x100
[ 1226.656190]  ? getname_flags+0x4a/0x1e0
[ 1226.656190]  ? __check_object_size+0xa8/0x16b
[ 1226.656190]  do_sys_open+0x184/0x220
[ 1226.656190]  do_syscall_64+0x5b/0x1a0
[ 1226.656190]  entry_SYSCALL_64_after_hwframe+0x65/0xca
[ 1226.656190] RIP: 0033:0x7f9234dd21d6
[ 1226.656191] Code: 89 54 24 08 e8 9b f4 ff ff 8b 74 24 0c 48 8b 3c 24 41 89 c0 44 8b 54 24 08 b8 01 01 00 00 89 f2 48 89 fe bf 9c ff ff ff 0f 05 <48> 3d 00 f0 ff ff 77 30 44 89 c7 89 44 24 08 e8 c6 f4 ff ff 8b 44
[ 1226.656191] RSP: 002b:00007f8f9afcd360 EFLAGS: 00000293 ORIG_RAX: 0000000000000101
[ 1226.656192] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f9234dd21d6
[ 1226.656192] RDX: 0000000000004000 RSI: 00007f8f30001568 RDI: 00000000ffffff9c
[ 1226.656192] RBP: 00007f8f30003150 R08: 0000000000000000 R09: 0000000000000000
[ 1226.656192] R10: 0000000000000000 R11: 0000000000000293 R12: 00007f8f9afcd4b8
[ 1226.656192] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000