Hang using lpfc driver when deleting devices and rescanning.
Issue
- Hang using
lpfc
driver when deleting devices and rescanning.
INFO: task lpfc_worker_1:600 blocked for more than 120 seconds.
Not tainted 2.6.32-504.3.3.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
lpfc_worker_1 D 0000000000000000 0 600 2 0x00000000
ffff880333f9f820 0000000000000046 0000000000000000 ffff880032e35928
ffff880032e35958 ffff880032e35928 00000080cb58461f 00000009166df0e0
ffff880334982080 000000010003dce1 ffff88033500e638 ffff880333f9ffd8
Call Trace:
[<ffffffff8152a885>] schedule_timeout+0x215/0x2e0
[<ffffffff8152a503>] wait_for_common+0x123/0x180
[<ffffffff81064b90>] ? default_wake_function+0x0/0x20
[<ffffffff8152a61d>] wait_for_completion+0x1d/0x20
[<ffffffff810983b1>] flush_cpu_workqueue+0x61/0x90
[<ffffffff81098460>] ? wq_barrier_func+0x0/0x20
[<ffffffff81098e84>] flush_workqueue+0x54/0x80
[<ffffffffa0231754>] fc_flush_work+0x24/0x50 [scsi_transport_fc]
[<ffffffffa0233f2a>] fc_remote_port_add+0x3a/0x7d0 [scsi_transport_fc]
[<ffffffffa0363f1e>] lpfc_nlp_state_cleanup+0x2fe/0x470 [lpfc]
[<ffffffffa0355013>] ? lpfc_els_rsp_prli_acc+0x113/0x1f0 [lpfc]
[<ffffffffa036414c>] lpfc_nlp_set_state+0xbc/0x1c0 [lpfc]
[<ffffffffa035e986>] ? lpfc_nlp_put+0x96/0x1b0 [lpfc]
[<ffffffffa037a75c>] ? lpfc_rcv_prli_prli_issue+0x1c/0x30 [lpfc]
[<ffffffffa037a98e>] lpfc_cmpl_prli_prli_issue+0x9e/0x200 [lpfc]
[<ffffffffa0378f3e>] lpfc_disc_state_machine+0x8e/0x200 [lpfc]
[<ffffffffa035a9c3>] lpfc_cmpl_els_prli+0x193/0x210 [lpfc]
[<ffffffffa0334d14>] ? lpfc_sli_free_hbq+0x34/0x80 [lpfc]
[<ffffffffa0348765>] lpfc_sli_sp_handle_rspiocb+0x445/0x6d0 [lpfc]
[<ffffffffa0343ad4>] ? __lpfc_sli_release_iocbq_s4+0x144/0x200 [lpfc]
[<ffffffffa03351d3>] ? lpfc_sli_release_iocbq+0x53/0x70 [lpfc]
[<ffffffffa0347a07>] ? lpfc_sli4_handle_received_buffer+0x4d7/0x8e0 [lpfc]
[<ffffffff8152997e>] ? thread_return+0x4e/0x7d0
[<ffffffffa0348b80>] lpfc_sli_handle_slow_ring_event_s4+0x190/0x280 [lpfc]
[<ffffffffa0334dff>] lpfc_sli_handle_slow_ring_event+0xf/0x20 [lpfc]
[<ffffffffa03670ec>] lpfc_do_work+0x89c/0x1760 [lpfc]
[<ffffffff8152997e>] ? thread_return+0x4e/0x7d0
[<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
[<ffffffffa0366850>] ? lpfc_do_work+0x0/0x1760 [lpfc]
[<ffffffff8109e66e>] kthread+0x9e/0xc0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffff8109e5d0>] ? kthread+0x0/0xc0
[<ffffffff8100c200>] ? child_rip+0x0/0x20
INFO: task scsi_wq_7:602 blocked for more than 120 seconds.
Not tainted 2.6.32-504.3.3.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
scsi_wq_7 D 0000000000000001 0 602 2 0x00000000
ffff880335f9bd20 0000000000000046 0000000000000000 ffffffff8160dd00
0000000000000000 ffff88033fc24300 0000007c11d4b632 ffffffff81063bf3
0000000000000000 0000000100038919 ffff880334982638 ffff880335f9bfd8
Call Trace:
[<ffffffff81063bf3>] ? perf_event_task_sched_out+0x33/0x70
[<ffffffff8152b306>] __mutex_lock_slowpath+0x96/0x210
[<ffffffff8152ae2b>] mutex_lock+0x2b/0x50
[<ffffffff8138c258>] scsi_scan_target+0x88/0xf0
[<ffffffffa023583d>] fc_scsi_scan_rport+0xbd/0xc0 [scsi_transport_fc]
[<ffffffffa0235780>] ? fc_scsi_scan_rport+0x0/0xc0 [scsi_transport_fc]
[<ffffffff81097fe0>] worker_thread+0x170/0x2a0
[<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81097e70>] ? worker_thread+0x0/0x2a0
[<ffffffff8109e66e>] kthread+0x9e/0xc0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffff8109e5d0>] ? kthread+0x0/0xc0
[<ffffffff8100c200>] ? child_rip+0x0/0x20
INFO: task fc_wq_7:603 blocked for more than 120 seconds.
Not tainted 2.6.32-504.3.3.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fc_wq_7 D 0000000000000002 0 603 2 0x00000000
ffff880335f9dd10 0000000000000046 0000000000000000 ffff88033a868aa0
ffff88033b3b8040 ffffffff8160dd00 0000007e5db453ee ffff880336014148
ffff880336014148 000000010003b40e ffff88033a869058 ffff880335f9dfd8
Call Trace:
[<ffffffff8152b306>] __mutex_lock_slowpath+0x96/0x210
[<ffffffff8150fc4c>] ? klist_next+0x4c/0xf0
[<ffffffff81386900>] ? target_unblock+0x0/0x50
[<ffffffff8152ae2b>] mutex_lock+0x2b/0x50
[<ffffffff8138dd18>] scsi_remove_device+0x28/0x50
[<ffffffff8138ded2>] scsi_remove_target+0x172/0x210
[<ffffffffa0235570>] ? fc_starget_delete+0x0/0x30 [scsi_transport_fc]
[<ffffffffa0235596>] fc_starget_delete+0x26/0x30 [scsi_transport_fc]
[<ffffffff81097fe0>] worker_thread+0x170/0x2a0
[<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81097e70>] ? worker_thread+0x0/0x2a0
[<ffffffff8109e66e>] kthread+0x9e/0xc0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffff8109e5d0>] ? kthread+0x0/0xc0
[<ffffffff8100c200>] ? child_rip+0x0/0x20
INFO: task multipathd:1265 blocked for more than 120 seconds.
Not tainted 2.6.32-504.3.3.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
multipathd D 0000000000000001 0 1265 1 0x00000000
ffff8803349f1eb8 0000000000000086 0000000000000000 ffffffff8104d18c
ffff8803349f1e38 0000000000000286 0000008093196884 000000023ac01800
ffff8803352c2040 000000010003d933 ffff8803352c25f8 ffff8803349f1fd8
Call Trace:
[<ffffffff8104d18c>] ? __do_page_fault+0x1ec/0x480
[<ffffffff8152a173>] io_schedule+0x73/0xc0
[<ffffffff811dc112>] wait_for_all_aios+0xd2/0x110
[<ffffffff81064b90>] ? default_wake_function+0x0/0x20
[<ffffffff811dd087>] io_destroy+0x87/0xe0
[<ffffffff811dd0fb>] sys_io_destroy+0x1b/0x60
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task rescan-scsi-bus:1672 blocked for more than 120 seconds.
Not tainted 2.6.32-504.3.3.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
rescan-scsi-b D 0000000000000001 0 1672 2060 0x00000000
ffff880337121cb8 0000000000000086 ffffea000b1f24e0 0000000000000008
ffff880337121ca8 ffffffff8114e603 0000000237121c78 00000000348e5f90
00000040ffffffff 0000000000000000 ffff8803343d1098 ffff880337121fd8
Call Trace:
[<ffffffff8114e603>] ? do_wp_page+0x493/0x920
[<ffffffff8152b306>] __mutex_lock_slowpath+0x96/0x210
[<ffffffff8152ae2b>] mutex_lock+0x2b/0x50
[<ffffffff8138c258>] scsi_scan_target+0x88/0xf0
[<ffffffffa0231955>] fc_user_scan+0x105/0x1b0 [scsi_transport_fc]
[<ffffffff8138d144>] store_scan+0xe4/0x120
[<ffffffff813674f0>] dev_attr_store+0x20/0x30
[<ffffffff8120a265>] sysfs_write_file+0xe5/0x170
[<ffffffff8118e058>] vfs_write+0xb8/0x1a0
[<ffffffff8118ea21>] sys_write+0x51/0x90
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
rport-6:0-11: blocked FC remote port time out: removing target and saving binding
The issue appears to be a process holding a mutex and waiting indefinitely for a request queue to drain.
Environment
- Red Hat Enterprise Linux 6
- Emulex HBA (lpfc)
Subscriber exclusive content
A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.