Server hang due to "block devices" with status "pending syncing" on driver layer

Issue

• Server hang, vmcore captured manually.

  VxVM vxdmp V-5-3-0 memory allocation failed for size =0x400000
  VxVM vxdmp V-5-3-0 memory allocation failed for size =0x400000
  VxVM vxdmp V-5-3-0 memory allocation failed for size =0x400000
  
  INFO: task jbd2/dm-0-8:26826 blocked for more than 120 seconds.
    Tainted: P           -- ------------    2.6.32-642.15.1.el6.x86_64 #1
  "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  jbd2/dm-0-8   D 0000000000000002     0 26826      2 0x00000000
  ffff8840027cbd20 0000000000000046 ffff8840027cbc70 ffffffff8106c512
  ffff884010c3f200 ffff8840b0c16f28 0000000000002be5 ffff8840110a0ab0
  ffff882013793520 ffffffff8160ca20 ffff8840110a1068 ffff8840027cbfd8
  Call Trace:
  [<ffffffff8106c512>] ? default_wake_function+0x12/0x20
  [<ffffffff810a6bce>] ? prepare_to_wait+0x4e/0x80
  [<ffffffffa05da80f>] jbd2_journal_commit_transaction+0x19f/0x14f0 [jbd2]
  [<ffffffff8108ef2c>] ? lock_timer_base+0x3c/0x70
  [<ffffffff810a68a0>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa05e0a38>] kjournald2+0xb8/0x220 [jbd2]
  [<ffffffff810a68a0>] ? autoremove_wake_function+0x0/0x40
  [<ffffffffa05e0980>] ? kjournald2+0x0/0x220 [jbd2]
  [<ffffffff810a640e>] kthread+0x9e/0xc0
  [<ffffffff8100c28a>] child_rip+0xa/0x20
  [<ffffffff810a6370>] ? kthread+0x0/0xc0
  [<ffffffff8100c280>] ? child_rip+0x0/0x20
  INFO: task flush-253:0:108387 blocked for more than 120 seconds.
    Tainted: P           -- ------------    2.6.32-642.15.1.el6.x86_64 #1
  "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
  flush-253:0   D 000000000000002d     0 108387      2 0x00000000
  ffff88698e167440 0000000000000046 0000000000000000 ffffffff81278ce4
  ffff88087ea9c960 ffff880973c1e100 0010f13934be52f2 ffffffffa0004f4f
  ffff8840136ae2d0 000000021c07da7d ffff88698690c5f8 ffff88698e167fd8
  Call Trace:
  [<ffffffff81278ce4>] ? blk_unplug+0x34/0x70
  [<ffffffffa0004f4f>] ? dm_table_unplug_all+0x5f/0x100 [dm_mod]
  [<ffffffff811d12c0>] ? sync_buffer+0x0/0x50
  [<ffffffff811d12c0>] ? sync_buffer+0x0/0x50
  [<ffffffff815493f3>] io_schedule+0x73/0xc0
  [<ffffffff811d1300>] sync_buffer+0x40/0x50
  [<ffffffff81549edf>] __wait_on_bit+0x5f/0x90
  [<ffffffff811d12c0>] ? sync_buffer+0x0/0x50
  [<ffffffff81549f88>] out_of_line_wait_on_bit+0x78/0x90
  [<ffffffff810a6920>] ? wake_bit_function+0x0/0x50
  [<ffffffff811d12b6>] __wait_on_buffer+0x26/0x30
  [<ffffffffa062ca44>] ext4_mb_init_cache+0x234/0x9f0 [ext4]
  [<ffffffffa062d82a>] ext4_mb_load_buddy+0x30a/0x390 [ext4]
  [<ffffffffa0004488>] ? dm_make_request+0x168/0x180 [dm_mod]
  [<ffffffffa0630d2f>] ext4_mb_regular_allocator+0x1af/0x410 [ext4]
  [<ffffffffa0629852>] ? ext4_mb_initialize_context+0x82/0x1d0 [ext4]
  [<ffffffffa06313a5>] ext4_mb_new_blocks+0x415/0x6a0 [ext4]
  [<ffffffffa062372a>] ? ext4_ext_find_extent+0x2ba/0x340 [ext4]
  [<ffffffff8112e6e7>] ? unlock_page+0x27/0x30
  [<ffffffffa0625a13>] ext4_ext_get_blocks+0x553/0x14d0 [ext4]
  [<ffffffffa06032c0>] ? noalloc_get_block_write+0x0/0x60 [ext4]
  [<ffffffffa06032c0>] ? noalloc_get_block_write+0x0/0x60 [ext4]
  [<ffffffff811d28c0>] ? block_write_full_page_endio+0xe0/0x120
  [<ffffffff811d1630>] ? end_buffer_async_write+0x0/0x190
  [<ffffffffa05fea30>] ? ext4_bh_delay_or_unwritten+0x0/0x30 [ext4]
  [<ffffffff812a1095>] ? radix_tree_gang_lookup_tag_slot+0x95/0xe0
  [<ffffffffa0602c09>] ext4_get_blocks+0xf9/0x2b0 [ext4]
  [<ffffffff81145fa6>] ? __pagevec_release+0x26/0x40
  [<ffffffffa0607311>] mpage_da_map_and_submit+0xa1/0x470 [ext4]
  [<ffffffffa0607f2e>] ext4_da_writepages+0x2ee/0x620 [ext4]
  [<ffffffff811439f1>] do_writepages+0x21/0x40
  [<ffffffff811c705d>] writeback_single_inode+0xdd/0x290
  [<ffffffff811c745d>] writeback_sb_inodes+0xbd/0x170
  [<ffffffff811c75bb>] writeback_inodes_wb+0xab/0x1b0
  [<ffffffff811c79b3>] wb_writeback+0x2f3/0x410
  [<ffffffff8108fbb2>] ? del_timer_sync+0x22/0x30
  [<ffffffff811c7c75>] wb_do_writeback+0x1a5/0x240
  [<ffffffff811c7d73>] bdi_writeback_task+0x63/0x1b0
  [<ffffffff810a6727>] ? bit_waitqueue+0x17/0xd0
  [<ffffffff81152bd0>] ? bdi_start_fn+0x0/0x100
  [<ffffffff81152c56>] bdi_start_fn+0x86/0x100
  [<ffffffff81152bd0>] ? bdi_start_fn+0x0/0x100
  [<ffffffff810a640e>] kthread+0x9e/0xc0
  [<ffffffff8100c28a>] child_rip+0xa/0x20
  [<ffffffff810a6370>] ? kthread+0x0/0xc0
  [<ffffffff8100c280>] ? child_rip+0x0/0x20