Hang using lpfc driver when deleting devices and rescanning.

Solution Verified - Updated -

Issue

  • Hang using lpfc driver when deleting devices and rescanning.
INFO: task lpfc_worker_1:600 blocked for more than 120 seconds.
       Not tainted 2.6.32-504.3.3.el6.x86_64 #1
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 lpfc_worker_1 D 0000000000000000     0   600      2 0x00000000
  ffff880333f9f820 0000000000000046 0000000000000000 ffff880032e35928
  ffff880032e35958 ffff880032e35928 00000080cb58461f 00000009166df0e0
  ffff880334982080 000000010003dce1 ffff88033500e638 ffff880333f9ffd8
 Call Trace:
  [<ffffffff8152a885>] schedule_timeout+0x215/0x2e0
  [<ffffffff8152a503>] wait_for_common+0x123/0x180
  [<ffffffff81064b90>] ? default_wake_function+0x0/0x20
  [<ffffffff8152a61d>] wait_for_completion+0x1d/0x20
  [<ffffffff810983b1>] flush_cpu_workqueue+0x61/0x90
  [<ffffffff81098460>] ? wq_barrier_func+0x0/0x20
  [<ffffffff81098e84>] flush_workqueue+0x54/0x80
  [<ffffffffa0231754>] fc_flush_work+0x24/0x50 [scsi_transport_fc]
  [<ffffffffa0233f2a>] fc_remote_port_add+0x3a/0x7d0 [scsi_transport_fc]
  [<ffffffffa0363f1e>] lpfc_nlp_state_cleanup+0x2fe/0x470 [lpfc]
  [<ffffffffa0355013>] ? lpfc_els_rsp_prli_acc+0x113/0x1f0 [lpfc]
  [<ffffffffa036414c>] lpfc_nlp_set_state+0xbc/0x1c0 [lpfc]
  [<ffffffffa035e986>] ? lpfc_nlp_put+0x96/0x1b0 [lpfc]
  [<ffffffffa037a75c>] ? lpfc_rcv_prli_prli_issue+0x1c/0x30 [lpfc]
  [<ffffffffa037a98e>] lpfc_cmpl_prli_prli_issue+0x9e/0x200 [lpfc]
  [<ffffffffa0378f3e>] lpfc_disc_state_machine+0x8e/0x200 [lpfc]
  [<ffffffffa035a9c3>] lpfc_cmpl_els_prli+0x193/0x210 [lpfc]
  [<ffffffffa0334d14>] ? lpfc_sli_free_hbq+0x34/0x80 [lpfc]
  [<ffffffffa0348765>] lpfc_sli_sp_handle_rspiocb+0x445/0x6d0 [lpfc]
  [<ffffffffa0343ad4>] ? __lpfc_sli_release_iocbq_s4+0x144/0x200 [lpfc]
  [<ffffffffa03351d3>] ? lpfc_sli_release_iocbq+0x53/0x70 [lpfc]
  [<ffffffffa0347a07>] ? lpfc_sli4_handle_received_buffer+0x4d7/0x8e0 [lpfc]
  [<ffffffff8152997e>] ? thread_return+0x4e/0x7d0
  [<ffffffffa0348b80>] lpfc_sli_handle_slow_ring_event_s4+0x190/0x280 [lpfc]
  [<ffffffffa0334dff>] lpfc_sli_handle_slow_ring_event+0xf/0x20 [lpfc]
  [<ffffffffa03670ec>] lpfc_do_work+0x89c/0x1760 [lpfc]
  [<ffffffff8152997e>] ? thread_return+0x4e/0x7d0
  [<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa0366850>] ? lpfc_do_work+0x0/0x1760 [lpfc]
  [<ffffffff8109e66e>] kthread+0x9e/0xc0
  [<ffffffff8100c20a>] child_rip+0xa/0x20
  [<ffffffff8109e5d0>] ? kthread+0x0/0xc0
  [<ffffffff8100c200>] ? child_rip+0x0/0x20
 INFO: task scsi_wq_7:602 blocked for more than 120 seconds.
       Not tainted 2.6.32-504.3.3.el6.x86_64 #1
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 scsi_wq_7     D 0000000000000001     0   602      2 0x00000000
  ffff880335f9bd20 0000000000000046 0000000000000000 ffffffff8160dd00
  0000000000000000 ffff88033fc24300 0000007c11d4b632 ffffffff81063bf3
  0000000000000000 0000000100038919 ffff880334982638 ffff880335f9bfd8
 Call Trace:
  [<ffffffff81063bf3>] ? perf_event_task_sched_out+0x33/0x70
  [<ffffffff8152b306>] __mutex_lock_slowpath+0x96/0x210
  [<ffffffff8152ae2b>] mutex_lock+0x2b/0x50
  [<ffffffff8138c258>] scsi_scan_target+0x88/0xf0
  [<ffffffffa023583d>] fc_scsi_scan_rport+0xbd/0xc0 [scsi_transport_fc]
  [<ffffffffa0235780>] ? fc_scsi_scan_rport+0x0/0xc0 [scsi_transport_fc]
  [<ffffffff81097fe0>] worker_thread+0x170/0x2a0
  [<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
  [<ffffffff81097e70>] ? worker_thread+0x0/0x2a0
  [<ffffffff8109e66e>] kthread+0x9e/0xc0
  [<ffffffff8100c20a>] child_rip+0xa/0x20
  [<ffffffff8109e5d0>] ? kthread+0x0/0xc0
  [<ffffffff8100c200>] ? child_rip+0x0/0x20
 INFO: task fc_wq_7:603 blocked for more than 120 seconds.
       Not tainted 2.6.32-504.3.3.el6.x86_64 #1
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 fc_wq_7       D 0000000000000002     0   603      2 0x00000000
  ffff880335f9dd10 0000000000000046 0000000000000000 ffff88033a868aa0
  ffff88033b3b8040 ffffffff8160dd00 0000007e5db453ee ffff880336014148
  ffff880336014148 000000010003b40e ffff88033a869058 ffff880335f9dfd8
 Call Trace:
  [<ffffffff8152b306>] __mutex_lock_slowpath+0x96/0x210
  [<ffffffff8150fc4c>] ? klist_next+0x4c/0xf0
  [<ffffffff81386900>] ? target_unblock+0x0/0x50
  [<ffffffff8152ae2b>] mutex_lock+0x2b/0x50
  [<ffffffff8138dd18>] scsi_remove_device+0x28/0x50
  [<ffffffff8138ded2>] scsi_remove_target+0x172/0x210
  [<ffffffffa0235570>] ? fc_starget_delete+0x0/0x30 [scsi_transport_fc]
  [<ffffffffa0235596>] fc_starget_delete+0x26/0x30 [scsi_transport_fc]
  [<ffffffff81097fe0>] worker_thread+0x170/0x2a0
  [<ffffffff8109eb00>] ? autoremove_wake_function+0x0/0x40
  [<ffffffff81097e70>] ? worker_thread+0x0/0x2a0
  [<ffffffff8109e66e>] kthread+0x9e/0xc0
  [<ffffffff8100c20a>] child_rip+0xa/0x20
  [<ffffffff8109e5d0>] ? kthread+0x0/0xc0
  [<ffffffff8100c200>] ? child_rip+0x0/0x20
 INFO: task multipathd:1265 blocked for more than 120 seconds.
       Not tainted 2.6.32-504.3.3.el6.x86_64 #1
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 multipathd    D 0000000000000001     0  1265      1 0x00000000
  ffff8803349f1eb8 0000000000000086 0000000000000000 ffffffff8104d18c
  ffff8803349f1e38 0000000000000286 0000008093196884 000000023ac01800
  ffff8803352c2040 000000010003d933 ffff8803352c25f8 ffff8803349f1fd8
 Call Trace:
  [<ffffffff8104d18c>] ? __do_page_fault+0x1ec/0x480
  [<ffffffff8152a173>] io_schedule+0x73/0xc0
  [<ffffffff811dc112>] wait_for_all_aios+0xd2/0x110
  [<ffffffff81064b90>] ? default_wake_function+0x0/0x20
  [<ffffffff811dd087>] io_destroy+0x87/0xe0
  [<ffffffff811dd0fb>] sys_io_destroy+0x1b/0x60
  [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
 INFO: task rescan-scsi-bus:1672 blocked for more than 120 seconds.
       Not tainted 2.6.32-504.3.3.el6.x86_64 #1
 "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
 rescan-scsi-b D 0000000000000001     0  1672   2060 0x00000000
  ffff880337121cb8 0000000000000086 ffffea000b1f24e0 0000000000000008
  ffff880337121ca8 ffffffff8114e603 0000000237121c78 00000000348e5f90
  00000040ffffffff 0000000000000000 ffff8803343d1098 ffff880337121fd8
 Call Trace:
  [<ffffffff8114e603>] ? do_wp_page+0x493/0x920
  [<ffffffff8152b306>] __mutex_lock_slowpath+0x96/0x210
  [<ffffffff8152ae2b>] mutex_lock+0x2b/0x50
  [<ffffffff8138c258>] scsi_scan_target+0x88/0xf0
  [<ffffffffa0231955>] fc_user_scan+0x105/0x1b0 [scsi_transport_fc]
  [<ffffffff8138d144>] store_scan+0xe4/0x120
  [<ffffffff813674f0>] dev_attr_store+0x20/0x30
  [<ffffffff8120a265>] sysfs_write_file+0xe5/0x170
  [<ffffffff8118e058>] vfs_write+0xb8/0x1a0
  [<ffffffff8118ea21>] sys_write+0x51/0x90
  [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
  rport-6:0-11: blocked FC remote port time out: removing target and saving binding

The issue appears to be a process holding a mutex and waiting indefinitely for a request queue to drain.

Environment

  • Red Hat Enterprise Linux 6
    • Emulex HBA (lpfc)

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In

New to Red Hat?

Learn more about Red Hat subscriptions

Using a Red Hat product through a public cloud?

How to access this content