A hard lockup occurs where a task is exitting and thus trying to take a tasklist_lock whereas another task has taken that tasklist_lock and writing OOM killing messages in a serial console.

Issue

• The kernel crashed due to a hard lockup.

Kernel panic - not syncing: Hard LOCKUP
Pid: 2608, comm: flush-253:0 Tainted: G           --L------------    2.6.32-754.9.1.el6.x86_64 #1
Call Trace:
 <NMI>  [<ffffffff815585ca>] ? panic+0xa7/0x18b
 [<ffffffff8100fa35>] ? show_trace+0x15/0x20
 [<ffffffff810f7600>] ? watchdog_timer_fn+0x0/0x230
 [<ffffffff811309aa>] ? __perf_event_overflow+0xaa/0x240
 [<ffffffff8101e728>] ? x86_perf_event_set_period+0xf8/0x180
 [<ffffffff81131004>] ? perf_event_overflow+0x14/0x20
 [<ffffffff81025c8c>] ? intel_pmu_handle_irq+0x21c/0x4a0
 [<ffffffff8155d0f3>] ? save_paranoid+0xf3/0x110
 [<ffffffff8155e83f>] ? perf_event_nmi_handler+0x3f/0xb0
 [<ffffffff81560350>] ? notifier_call_chain+0x50/0x80
 [<ffffffff8155d0e7>] ? save_paranoid+0xe7/0x110
 [<ffffffff815603ba>] ? atomic_notifier_call_chain+0x1a/0x20
 [<ffffffff810b12ee>] ? notify_die+0x2e/0x30
 [<ffffffff8155dea9>] ? do_nmi+0xd9/0x360
 [<ffffffff8155d781>] ? nmi+0x131/0x193
 [<ffffffff811cf9c0>] ? bdi_writeback_thread+0x0/0x230
 [<ffffffff812b2049>] ? __write_lock_failed+0x9/0x20
 <<EOE>>  [<ffffffff8155c337>] ? _write_lock+0x17/0x20
 [<ffffffff8107e08b>] ? tasklist_write_lock_irq+0x3b/0x50
 [<ffffffff81085c11>] ? do_exit+0x201/0x860
 [<ffffffff811cf9c0>] ? bdi_writeback_thread+0x0/0x230
 [<ffffffff810aaa21>] ? kthread+0x91/0xc0
 [<ffffffff815657d0>] ? child_rip+0x20/0x30
 [<ffffffff810aa990>] ? kthread+0x0/0xc0
 [<ffffffff815657b0>] ? child_rip+0x0/0x30

• This task is exitting and thus trying to take a tasklist_lock that has been taken by another task on CPU 8.

PID: 15986  TASK: ffff88023dc36ab0  CPU: 8   COMMAND: "splunkd"
 #0 [ffff8818d4809e90] crash_nmi_callback at ffffffff8103728c
 #1 [ffff8818d4809ea0] notifier_call_chain at ffffffff81560350
 #2 [ffff8818d4809ee0] atomic_notifier_call_chain at ffffffff815603ba
 #3 [ffff8818d4809ef0] notify_die at ffffffff810b12ee
 #4 [ffff8818d4809f20] do_nmi at ffffffff8155dea9
 #5 [ffff8818d4809f50] nmi at ffffffff8155d781
    [exception RIP: io_serial_in+0x16]
    RIP: ffffffff81383e56  RSP: ffff880cc7e673b8  RFLAGS: 00000002
    RAX: ffffffff81383e00  RBX: ffffffff8202f878  RCX: 0000000000000000
    RDX: 00000000000002fd  RSI: 0000000000000005  RDI: ffffffff8202f878
    RBP: ffff880cc7e673b8   R8: 0000000000407a6e   R9: 0000000000000000
    R10: 0000000000000000  R11: 0000000000000198  R12: 0000000000002685
    R13: 0000000000000020  R14: 0000000000000000  R15: ffffffff81384610
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0000
--- <NMI exception stack> ---
 #6 [ffff880cc7e673b8] io_serial_in at ffffffff81383e56
 #7 [ffff880cc7e673c0] wait_for_xmitr at ffffffff81384571
 #8 [ffff880cc7e673f0] serial8250_console_putchar at ffffffff81384636
 #9 [ffff880cc7e67410] uart_console_write at ffffffff8137fd70
#10 [ffff880cc7e67450] serial8250_console_write at ffffffff813849e9
#11 [ffff880cc7e674a0] __call_console_drivers at ffffffff81080758
#12 [ffff880cc7e674d0] _call_console_drivers at ffffffff810807ba
#13 [ffff880cc7e674f0] release_console_sem at ffffffff81080e08
#14 [ffff880cc7e67530] vprintk at ffffffff81081518
#15 [ffff880cc7e675d0] vprintk_default at ffffffff8108183e
#16 [ffff880cc7e675e0] printk at ffffffff81558703
#17 [ffff880cc7e67640] __show_free_areas at ffffffff81140867
#18 [ffff880cc7e67830] show_mem at ffffffff812ac348
#19 [ffff880cc7e678b0] dump_header at ffffffff81136686
#20 [ffff880cc7e67920] oom_kill_process at ffffffff81136af2
#21 [ffff880cc7e67990] out_of_memory at ffffffff81136f30
#22 [ffff880cc7e67a10] __alloc_pages_nodemask at ffffffff811439f1
             ...

• The system is running out of memory and 100% swapping.

                 PAGES        TOTAL      PERCENTAGE
    TOTAL MEM  49554126       189 GB         ----
         FREE   104055     406.5 MB    0% of TOTAL MEM
         USED  49450071     188.6 GB   99% of TOTAL MEM
       SHARED     7921      30.9 MB    0% of TOTAL MEM
      BUFFERS      226       904 KB    0% of TOTAL MEM
       CACHED  8068823      30.8 GB   16% of TOTAL MEM
         SLAB   120578       471 MB    0% of TOTAL MEM

   TOTAL HUGE        0            0         ----
    HUGE FREE        0            0    0% of TOTAL HUGE

   TOTAL SWAP  2047999       7.8 GB         ----
    SWAP USED  2047999       7.8 GB  100% of TOTAL SWAP
    SWAP FREE        0            0    0% of TOTAL SWAP

 COMMIT LIMIT  26825062     102.3 GB         ----
    COMMITTED  37173555     141.8 GB  138% of TOTAL LIMIT

• Massive page allocation failures are encountered around the time of the crash:

swapper: page allocation failure. order:0, mode:0x20
Pid: 0, comm: swapper Tainted: G           --L------------    2.6.32-754.9.1.el6.x86_64 #1
Call Trace:
 <IRQ>  [<ffffffff81143891>] ? __alloc_pages_nodemask+0x7e1/0x960
 [<ffffffff811431d9>] ? __alloc_pages_nodemask+0x129/0x960
 [<ffffffff81185f92>] ? kmem_getpages+0x62/0x170
 [<ffffffff81186baa>] ? fallback_alloc+0x1ba/0x270
 [<ffffffff81186607>] ? cache_grow+0x2d7/0x320
 [<ffffffff81186929>] ? ____cache_alloc_node+0x99/0x160
 [<ffffffff81187bb0>] ? kmem_cache_alloc_node_trace+0x90/0x210
 [<ffffffff81187ddd>] ? __kmalloc_node+0x4d/0x60
 [<ffffffff8147beba>] ? __alloc_skb+0x7a/0x190
 [<ffffffff8147c4e4>] ? __netdev_alloc_skb+0x24/0x50
 [<ffffffffa019adb5>] ? tg3_poll_work+0x9d5/0x1010 [tg3]
 [<ffffffffa019b441>] ? tg3_poll_msix+0x51/0x180 [tg3]
 [<ffffffff8148e6f9>] ? net_rx_action+0x109/0x320
 [<ffffffff8108973a>] ? __do_softirq+0xea/0x240
 [<ffffffff810f8336>] ? handle_IRQ_event+0x66/0x180
 [<ffffffff815658cc>] ? call_softirq+0x1c/0x30
 [<ffffffff8100e4c5>] ? do_softirq+0x65/0xa0
 [<ffffffff810893cd>] ? irq_exit+0x8d/0xa0
 [<ffffffff81566725>] ? do_IRQ+0x75/0xf0
 [<ffffffff81564e93>] ? ret_from_intr+0x0/0x11
 <EOI>  [<ffffffff8130828e>] ? intel_idle+0x12e/0x250
           ...