Hard lockup on virtual system while in pvscsi_queue()

Issue

• Hard lockup on virtual system while in pvscsi_queue() with console messages similar to:

[235854.036781] NMI watchdog: Watchdog detected hard LOCKUP on cpu 0
[235854.036787] Modules linked in
....
[235854.036836] CPU: 0 PID: 142432 Comm: kworker/0:2 Kdump: loaded Not tainted 5.14.0-570.30.1.el9_6.x86_64 #1
[235854.036839] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[235854.036842] Workqueue: xfs-sync/dm-8 xfs_log_worker [xfs]
[235854.036960] RIP: 0010:native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.036968] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d e9 d0 0f 00 00 f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00
[235854.036970] RSP: 0018:ffffa0d5c2bb79f8 EFLAGS: 00000002
[235854.036972] RAX: 0000000000000001 RBX: ffff8f67d4c79c00 RCX: 000000000000000a
[235854.036974] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8f67d3204030
[235854.036975] RBP: ffff8f67d3204030 R08: 0000000000000010 R09: 0000000000000000
[235854.036977] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000286
[235854.036978] R13: ffff8f67d3204000 R14: ffffa0d5c2bb7b10 R15: ffff8f67d302e978
[235854.036980] FS:  0000000000000000(0000) GS:ffff8f7aade00000(0000) knlGS:0000000000000000
[235854.036982] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[235854.036984] CR2: 00007fffcb93b9f8 CR3: 000000018de88000 CR4: 0000000000350ef0
[235854.037003] Call Trace:
[235854.037006]  <NMI>
[235854.037007]  ? srso_return_thunk+0x5/0x5f
[235854.037010]  ? show_trace_log_lvl+0x26e/0x2df
[235854.037016]  ? show_trace_log_lvl+0x26e/0x2df
[235854.037021]  ? _raw_spin_lock_irqsave+0x30/0x40
[235854.037026]  ? watchdog_overflow_callback.cold+0x1e/0x70
[235854.037029]  ? __perf_event_overflow+0x115/0x320
[235854.037034]  ? srso_return_thunk+0x5/0x5f
[235854.037037]  ? x86_pmu_handle_irq+0x14a/0x1d0
[235854.037052]  ? amd_pmu_handle_irq+0x42/0xb0
[235854.037055]  ? perf_event_nmi_handler+0x28/0x50
[235854.037059]  ? nmi_handle+0x5e/0x120
[235854.037064]  ? default_do_nmi+0x40/0x130
[235854.037067]  ? exc_nmi+0x100/0x180
[235854.037070]  ? end_repeat_nmi+0xf/0x60
[235854.037084]  ? native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.037088]  ? native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.037091]  ? native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.037095]  </NMI>
[235854.037096]  <TASK>
[235854.037098]  _raw_spin_lock_irqsave+0x30/0x40
[235854.037102]  pvscsi_queue+0x22/0x242 [vmw_pvscsi]
[235854.037109]  ? srso_return_thunk+0x5/0x5f
[235854.037111]  ? srso_return_thunk+0x5/0x5f
[235854.037114]  scsi_dispatch_cmd+0x90/0x240
[235854.037118]  scsi_queue_rq+0x1ad/0x610
[235854.037122]  blk_mq_dispatch_rq_list+0x290/0x4b0
[235854.037127]  ? sbitmap_get+0x11/0xe0
[235854.037132]  __blk_mq_do_dispatch_sched+0xb8/0x310
[235854.037139]  __blk_mq_sched_dispatch_requests+0x14d/0x190
[235854.037141]  ? srso_return_thunk+0x5/0x5f
[235854.037144]  blk_mq_sched_dispatch_requests+0x26/0x60
[235854.037147]  blk_mq_run_hw_queue+0x154/0x1b0
[235854.037150]  blk_mq_dispatch_plug_list+0x17e/0x310
[235854.037156]  blk_mq_flush_plug_list+0x40/0x1e0
[235854.037159]  __blk_flush_plug+0xf1/0x150
[235854.037165]  __submit_bio+0x191/0x230
[235854.037170]  __submit_bio_noacct+0x7e/0x1e0
[235854.037175]  xlog_state_release_iclog+0xe6/0x1c0 [xfs]
[235854.037289]  xfs_log_force+0x172/0x230 [xfs]
[235854.037391]  xfs_log_worker+0x3b/0xd0 [xfs]
[235854.037489]  process_one_work+0x197/0x380
[235854.037494]  worker_thread+0x2fe/0x410
[235854.037497]  ? __pfx_worker_thread+0x10/0x10
[235854.037499]  kthread+0xe0/0x100
[235854.037503]  ? __pfx_kthread+0x10/0x10
[235854.037508]  ret_from_fork+0x2c/0x50
[235854.037515]  </TASK>
[235854.037517] Kernel panic - not syncing: Hard LOCKUP
[235854.037519] CPU: 0 PID: 142432 Comm: kworker/0:2 Kdump: loaded Not tainted 5.14.0-570.30.1.el9_6.x86_64 #1
[235854.037522] Hardware name: VMware, Inc. VMware Virtual Platform/440BX Desktop Reference Platform, BIOS 6.00 11/12/2020
[235854.037524] Workqueue: xfs-sync/dm-8 xfs_log_worker [xfs]
[235854.037626] Call Trace:
[235854.037628]  <NMI>
[235854.037630]  dump_stack_lvl+0x34/0x48
[235854.037636]  panic+0x107/0x2bb
[235854.037641]  nmi_panic.cold+0xc/0xc
[235854.037643]  watchdog_overflow_callback.cold+0x5c/0x70
[235854.037646]  __perf_event_overflow+0x115/0x320
[235854.037650]  ? srso_return_thunk+0x5/0x5f
[235854.037654]  x86_pmu_handle_irq+0x14a/0x1d0
[235854.037668]  amd_pmu_handle_irq+0x42/0xb0
[235854.037671]  perf_event_nmi_handler+0x28/0x50
[235854.037675]  nmi_handle+0x5e/0x120
[235854.037679]  default_do_nmi+0x40/0x130
[235854.037683]  exc_nmi+0x100/0x180
[235854.037686]  end_repeat_nmi+0xf/0x60
[235854.037689] RIP: 0010:native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.037692] Code: 54 55 48 89 fd 53 66 90 ba 01 00 00 00 8b 45 00 85 c0 75 14 f0 0f b1 55 00 85 c0 75 f0 5b 5d 41 5c 41 5d e9 d0 0f 00 00 f3 90 <eb> e1 81 fe 00 01 00 00 74 52 40 30 f6 85 f6 75 75 f0 0f ba 6d 00
[235854.037694] RSP: 0018:ffffa0d5c2bb79f8 EFLAGS: 00000002
[235854.037696] RAX: 0000000000000001 RBX: ffff8f67d4c79c00 RCX: 000000000000000a
[235854.037698] RDX: 0000000000000001 RSI: 0000000000000001 RDI: ffff8f67d3204030
[235854.037699] RBP: ffff8f67d3204030 R08: 0000000000000010 R09: 0000000000000000
[235854.037701] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000286
[235854.037702] R13: ffff8f67d3204000 R14: ffffa0d5c2bb7b10 R15: ffff8f67d302e978
[235854.037708]  ? native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.037712]  ? native_queued_spin_lock_slowpath+0x32/0x2b0
[235854.037716]  </NMI>
[235854.037717]  <TASK>
[235854.037718]  _raw_spin_lock_irqsave+0x30/0x40
[235854.037721]  pvscsi_queue+0x22/0x242 [vmw_pvscsi]
[235854.037727]  ? srso_return_thunk+0x5/0x5f
[235854.037729]  ? srso_return_thunk+0x5/0x5f
[235854.037733]  scsi_dispatch_cmd+0x90/0x240
[235854.037736]  scsi_queue_rq+0x1ad/0x610
[235854.037740]  blk_mq_dispatch_rq_list+0x290/0x4b0
[235854.037744]  ? sbitmap_get+0x11/0xe0
[235854.037748]  __blk_mq_do_dispatch_sched+0xb8/0x310
[235854.037753]  __blk_mq_sched_dispatch_requests+0x14d/0x190
[235854.037756]  ? srso_return_thunk+0x5/0x5f
[235854.037759]  blk_mq_sched_dispatch_requests+0x26/0x60
[235854.037761]  blk_mq_run_hw_queue+0x154/0x1b0
[235854.037765]  blk_mq_dispatch_plug_list+0x17e/0x310
[235854.037770]  blk_mq_flush_plug_list+0x40/0x1e0
[235854.037773]  __blk_flush_plug+0xf1/0x150
[235854.037778]  __submit_bio+0x191/0x230
[235854.037782]  __submit_bio_noacct+0x7e/0x1e0
[235854.037787]  xlog_state_release_iclog+0xe6/0x1c0 [xfs]
[235854.037895]  xfs_log_force+0x172/0x230 [xfs]
[235854.037994]  xfs_log_worker+0x3b/0xd0 [xfs]
[235854.038092]  process_one_work+0x197/0x380
[235854.038096]  worker_thread+0x2fe/0x410
[235854.038099]  ? __pfx_worker_thread+0x10/0x10
[235854.038101]  kthread+0xe0/0x100
[235854.038104]  ? __pfx_kthread+0x10/0x10
[235854.038107]  ret_from_fork+0x2c/0x50
[235854.038114]  </TASK>