The kernel crashes due to double-fault with a message, "BUG: stack guard page was hit at 00000000cf391aa4 (stack is 00000000bd2d3d06..000000006ca6d476)"

Solution Unverified - Updated -

Issue

  • The kernel crashes due to double-fault with a message, "BUG: stack guard page was hit at 00000000cf391aa4 (stack is 00000000bd2d3d06..000000006ca6d476)"
[409170.788752] BUG: stack guard page was hit at 00000000cf391aa4 (stack is 00000000bd2d3d06..000000006ca6d476)
[409170.788753] kernel stack overflow (double-fault): 0000 [#1] SMP PTI
[409170.788754] CPU: 50 PID: 195812 Comm: vhost-195805 Kdump: loaded Tainted: G           OE    --------- -  - 4.18.0-305.72.1.el8_4.x86_64 #1
[409170.788755] Hardware name: HPE ProLiant DL380 Gen10/ProLiant DL380 Gen10, BIOS U30 09/03/2021
[409170.788756] RIP: 0010:vr_forward+0x101/0xd10 [vrouter]
[409170.788758] Code: 50 80 00 00 00 48 89 84 24 b0 00 00 00 48 89 94 24 b8 00 00 00 48 8d 74 24 28 c7 44 24 68 00 00 00 00 c7 44 24 78 00 00 00 00 <e8> 1a a5 00 00 48 89 c6 48 85 c0 0f 84 34 01 00 00 f6 44 24 5a 01
[409170.788758] RSP: 0018:ffff99469fdbfff0 EFLAGS: 00010202
[409170.788759] RAX: 00000000b9887c0a RBX: ffff8caa0adc050e RCX: 0000000000000026
[409170.788760] RDX: ffff99469fdc3b28 RSI: ffff99469fdc0018 RDI: 0000000000000013
[409170.788761] RBP: ffff99469fdc0120 R08: ffff8c3a2c36e028 R09: 0000000000000000
[409170.788761] R10: 00000000ffffffff R11: 0000000000000012 R12: ffff8c3a2c36e028
[409170.788762] R13: ffff99469fdc3b28 R14: ffff99469fdc3b28 R15: ffff8c9e321e4000
[409170.788763] FS:  0000000000000000(0000) GS:ffff8cb93ff80000(0000) knlGS:0000000000000000
[409170.788763] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[409170.788764] CR2: ffff99469fdbffe8 CR3: 00000075506d0001 CR4: 00000000007726e0
[409170.788765] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[409170.788765] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[409170.788766] PKRU: 55555554
[409170.788766] Call Trace:
[409170.788766]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788767]  nh_output+0x47/0x510 [vrouter]
[409170.788768]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788768]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788769]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788769]  nh_output+0x47/0x510 [vrouter]
[409170.788770]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788770]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788771]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788771]  nh_output+0x47/0x510 [vrouter]
[409170.788772]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788772]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788772]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788773]  nh_output+0x47/0x510 [vrouter]
[409170.788773]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788774]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788774]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788775]  nh_output+0x47/0x510 [vrouter]
[409170.788775]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788776]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788776]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788777]  nh_output+0x47/0x510 [vrouter]
[409170.788777]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788778]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788778]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788779]  nh_output+0x47/0x510 [vrouter]
[409170.788779]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788780]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788781]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788781]  nh_output+0x47/0x510 [vrouter]
[409170.788782]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788782]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788783]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788783]  nh_output+0x47/0x510 [vrouter]
[409170.788784]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788784]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788785]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788785]  nh_output+0x47/0x510 [vrouter]
[409170.788785]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788786]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788786]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788787]  nh_output+0x47/0x510 [vrouter]
[409170.788787]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788788]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788789]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788790]  nh_output+0x47/0x510 [vrouter]
[409170.788790]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788791]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788791]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788792]  nh_output+0x47/0x510 [vrouter]
[409170.788792]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788793]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788793]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788794]  nh_output+0x47/0x510 [vrouter]
[409170.788795]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788795]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788796]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788796]  nh_output+0x47/0x510 [vrouter]
[409170.788796]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788797]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788797]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788798]  nh_output+0x47/0x510 [vrouter]
[409170.788798]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788798]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788799]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788799]  nh_output+0x47/0x510 [vrouter]
[409170.788800]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788800]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788801]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788801]  nh_output+0x47/0x510 [vrouter]
[409170.788801]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788802]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788802]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788803]  nh_output+0x47/0x510 [vrouter]
[409170.788803]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788803]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788804]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788804]  nh_output+0x47/0x510 [vrouter]
[409170.788804]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788805]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788805]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788806]  nh_output+0x47/0x510 [vrouter]
[409170.788806]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788807]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788807]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788807]  nh_output+0x47/0x510 [vrouter]
[409170.788808]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788809]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788809]  ? apic_timer_interrupt+0xa/0x20
[409170.788810]  ? call_function_single_interrupt+0xa/0x20
[409170.788810]  ? call_function_single_interrupt+0xa/0x20
[409170.788811]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788811]  nh_output+0x47/0x510 [vrouter]
[409170.788812]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788812]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788813]  ? dev_hard_start_xmit+0x96/0x1e0
[409170.788813]  ? __qdisc_run+0x83/0x550
[409170.788814]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788814]  nh_output+0x47/0x510 [vrouter]
[409170.788815]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788815]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788815]  ? __dev_queue_xmit+0x45a/0x920
[409170.788816]  ? dev_hard_start_xmit+0x96/0x1e0
[409170.788816]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788817]  nh_output+0x47/0x510 [vrouter]
[409170.788817]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788818]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788818]  ? dev_hard_start_xmit+0x96/0x1e0
[409170.788819]  ? __dev_queue_xmit+0x45a/0x920
[409170.788819]  nh_mpls_udp_tunnel+0x2ee/0x670 [vrouter]
[409170.788820]  nh_output+0x47/0x510 [vrouter]
[409170.788820]  ? __mtrie_lookup+0xf9/0x180 [vrouter]
[409170.788821]  vr_forward+0x14f/0xd10 [vrouter]
[409170.788821]  ? dev_hard_start_xmit+0x96/0x1e0
[409170.788822]  ? vr_flow_forward+0xee/0xd50 [vrouter]
[409170.788822]  ? vr_htable_find_hentry+0x2a8/0x350 [vrouter]
[409170.788823]  vr_ip_input+0xa0/0x4d0 [vrouter]
[409170.788823]  ? vr_flow_lookup+0x1ab/0xc90 [vrouter]
[409170.788824]  vr_l3_input+0x44/0x50 [vrouter]
[409170.788824]  nh_l2_rcv+0x146/0x950 [vrouter]
[409170.788825]  ? __bridge_lookup.isra.11.part.12+0x57/0x80 [vrouter]
[409170.788826]  nh_output+0x47/0x510 [vrouter]
[409170.788826]  vr_bridge_input+0x36a/0x1d70 [vrouter]
[409170.788827]  ? get_page_from_freelist+0x1118/0x1360
[409170.788827]  vr_virtual_input+0xb2/0x350 [vrouter]
[409170.788828]  ? enqueue_entity+0xdc/0x640
[409170.788828]  ? sched_clock+0x5/0x10
[409170.788829]  ? enqueue_task_fair+0x93/0x710
[409170.788829]  ? remove_entity_load_avg+0x31/0x80
[409170.788830]  vm_rx+0x173/0x1c0 [vrouter]
[409170.788830]  ? try_to_wake_up+0x374/0x580
[409170.788831]  linux_rx_handler+0x3ba/0xb20 [vrouter]
[409170.788831]  ? vmx_deliver_posted_interrupt+0x38/0x110 [kvm_intel]
[409170.788832]  ? __apic_accept_irq+0x189/0x2e0 [kvm]
[409170.788832]  __netif_receive_skb_core+0x287/0xc50
[409170.788833]  ? __skb_get_hash_symmetric+0x55/0x1e0
[409170.788833]  ? __skb_get_hash_symmetric+0x184/0x1e0
[409170.788834]  ? recalibrate_cpu_khz+0x10/0x10
[409170.788834]  netif_receive_skb_internal+0x3d/0xb0
[409170.788835]  tun_sendmsg+0x3cd/0x750 [tun]
[409170.788835]  vhost_tx_batch.isra.19+0x64/0x1e0 [vhost_net]
[409170.788836]  handle_tx_copy+0x179/0x580 [vhost_net]
[409170.788836]  ? vhost_dev_init+0x270/0x270 [vhost]
[409170.788837]  handle_tx+0xa7/0xe0 [vhost_net]
[409170.788837]  vhost_worker+0x91/0xd0 [vhost]
[409170.788838]  kthread+0x116/0x130
[409170.788838]  ? kthread_flush_work_fn+0x10/0x10
[409170.788839]  ret_from_fork+0x35/0x40
[409170.788839] Modules linked in: binfmt_misc tcp_diag udp_diag inet_diag vhost_net vhost vhost_iotlb tap tun overlay vrouter(OE) 8021q garp mrp bonding openvswitch nf_conncount nf_nat nf_log_ipv6 nf_log_ipv4 nf_log_common nft_limit nft_counter xt_LOG xt_limit xt_multiport xt_comment xt_state xt_conntrack nft_compat nf_tables nfnetlink vfat fat dm_service_time dm_multipath dm_mod rpcrdma sunrpc rdma_ucm ib_srpt ib_isert iscsi_target_mod target_core_mod ib_iser libiscsi scsi_transport_iscsi ib_umad rdma_cm ib_ipoib iw_cm ib_cm intel_rapl_msr intel_rapl_common isst_if_common nfit libnvdimm x86_pkg_temp_thermal intel_powerclamp coretemp kvm_intel kvm irqbypass ipmi_ssif crct10dif_pclmul crc32_pclmul ghash_clmulni_intel rapl intel_cstate intel_uncore mlx5_ib pcspkr ib_uverbs cdc_eem mei_me ib_core usbnet ses enclosure mii acpi_ipmi mei lpc_ich hpwdt hpilo ioatdma ipmi_si wmi ipmi_devintf ipmi_msghandler acpi_tad acpi_power_meter xfs sd_mod sg lpfc mlx5_core mgag200 drm_kms_helper nvmet_fc nvmet
[409170.788868]  syscopyarea sysfillrect sysimgblt nvme_fc fb_sys_fops nvme_fabrics drm igb smartpqi mlxfw nvme_core pci_hyperv_intf tls scsi_transport_sas tg3 dca uas t10_pi scsi_transport_fc usb_storage.
[409170.788874] Lost 11 message(s)!

Environment

  • Red Hat Enterprise Linux 8.4 - kernel-4.18.0-305.72.1.el8_4
  • Red Hat OpenStack Platform 16.2
  • 3rd party module named "vrouter"

Subscriber exclusive content

A Red Hat subscription provides unlimited access to our knowledgebase, tools, and much more.

Current Customers and Partners

Log in for full access

Log In

New to Red Hat?

Learn more about Red Hat subscriptions

Using a Red Hat product through a public cloud?

How to access this content