The system crashed with a panic string "kernel BUG at security/selinux/avc.c:167!"
Environment
- Red Hat Enterprise Linux
Issue
- The kernel panics with panic string
kernel BUG at security/selinux/avc.c:167!
Resolution
- Red Hat neither ships nor supports this module. Engage the respective vendor of the module
vxiofor further investigation.
Root Cause
A task running a code from a 3rd party module vxio caused the memory corruption in sk_security_struct and memory around it.
Diagnostic Steps
Pre-requisites
-
Deploy kdump in Order to Collect a vmcore:
- Vmcore analyis is required to determine if you are being impacted by this issue. This first requires that a vmcore is dumped successfully.
- If the
kexec-toolspackage is absent or thekdumpservice is inactive, please reference the following article to install, enable, start, and configure kdump:
How to troubleshoot kernel crashes, hangs, or reboots with kdump on Red Hat Enterprise Linux
-
Prepare crash Environment for vmcore Analysis:
- Please reference the following article to set up a vmcore analysis environment:
How to set up a vmcore analysis environment?
- Please reference the following article to set up a vmcore analysis environment:
Vmcore Analysis
-
System Information:
crash> sys |grep -eREL -ePAN -eLOAD LOAD AVERAGE: 1.55, 3.01, 2.06 RELEASE: 3.10.0-1160.59.1.el7.x86_64 PANIC: "kernel BUG at security/selinux/avc.c:167!" crash> sys -i |head -5 DMI_BIOS_VENDOR: Phoenix Technologies LTD DMI_BIOS_VERSION: 6.00 DMI_BIOS_DATE: 12/12/2018 DMI_SYS_VENDOR: VMware, Inc. DMI_PRODUCT_NAME: VMware Virtual Platform -
Backtrace of the panic task:
crash> bt PID: 16904 TASK: ffff8a43b2828000 CPU: 29 COMMAND: "vxnetd" #0 [ffff8a45ca717150] machine_kexec at ffffffffa18662f4 #1 [ffff8a45ca7171b0] __crash_kexec at ffffffffa1922a32 #2 [ffff8a45ca717280] crash_kexec at ffffffffa1922b20 #3 [ffff8a45ca717298] oops_end at ffffffffa1f91798 #4 [ffff8a45ca7172c0] die at ffffffffa1830a7b #5 [ffff8a45ca7172f0] do_trap at ffffffffa1f90ee0 #6 [ffff8a45ca717340] do_invalid_op at ffffffffa182d2a4 #7 [ffff8a45ca7173f0] invalid_op at ffffffffa1f9d2ee [exception RIP: avc_audit_post_callback+0x16b] RIP: ffffffffa1b0b5ab RSP: ffff8a45ca7174a8 RFLAGS: 00010206 RAX: ffff8a45f5f5f901 RBX: ffff8a33f0ac2f40 RCX: 000000000557297f RDX: 000000000557297e RSI: 000000000000002a RDI: ffff8a237fc03b00 RBP: ffff8a45ca7174e8 R8: 000000000001f0a0 R9: ffffffffa1b0b5a6 R10: ffff8a45f755f0a0 R11: ffffe10e4fd7d7c0 R12: ffff8a45ca717690 R13: 0000000000004e47 R14: 0000000053494e47 R15: 0000000000000001 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 #8 [ffff8a45ca7174a0] avc_audit_post_callback at ffffffffa1b0b5a6 #9 [ffff8a45ca7174f0] common_lsm_audit at ffffffffa1b2ad10 #10 [ffff8a45ca717580] slow_avc_audit at ffffffffa1b0b8ba #11 [ffff8a45ca7175d8] avc_has_perm_flags at ffffffffa1b0c0f2 #12 [ffff8a45ca717688] sock_has_perm at ffffffffa1b0c895 #13 [ffff8a45ca7176f8] selinux_socket_recvmsg at ffffffffa1b0c993 #14 [ffff8a45ca717708] security_socket_recvmsg at ffffffffa1b09d9c #15 [ffff8a45ca717718] sock_recvmsg at ffffffffa1e394b1 #16 [ffff8a45ca717880] kmsg_sys_rcv at ffffffffc2d9df39 [vxio] #17 [ffff8a45ca717918] nmcom_get_next_mblk at ffffffffc30099b3 [vxio] #18 [ffff8a45ca717960] nmcom_get_data_msg at ffffffffc3004f20 [vxio] #19 [ffff8a45ca717a28] nmcom_get_next_msg at ffffffffc3005307 [vxio] #20 [ffff8a45ca717a88] nmcom_wait_msg_tcp at ffffffffc30053f3 [vxio] #21 [ffff8a45ca717ae0] nmcom_server_proc_tcp at ffffffffc301841b [vxio] #22 [ffff8a45ca717b80] nmcom_server_main_tcp at ffffffffc3019cc3 [vxio] #23 [ffff8a45ca717ec8] kthread at ffffffffa18c5e61 -
The corruption was found by a task on CPU 29 in a
sk_security_structthat belonged to a socket it had open, it failed an array index check in the code which caused the system crash:crash> bt -f [..] #8 [ffff8a45ca7174a0] avc_audit_post_callback at ffffffffa1b0b5a6 ffff8a45ca7174a8: 00000021ca7174b8 ffff8a45f5f5f600 rbx=audit_buffer *ab ffff8a45ca7174b8: 0000000095458ec0 ffff8a33f0ac2f40 r12=a r13 ffff8a45ca7174c8: ffff8a45ca717690 ffffffffa1b0b440 r14 r15 ffff8a45ca7174d8: ffff8a33e29aa940 000000000000acf9 rbp ffff8a45ca7174e8: ffff8a45ca717578 ffffffffa1b2ad10 #9 [ffff8a45ca7174f0] common_lsm_audit at ffffffffa1b2ad10 [..] crash> audit_buffer ffff8a33f0ac2f40 struct audit_buffer { list = { next = 0xdead000000000100, prev = 0xdead000000000200 }, skb = 0xffff8a33da182300, ctx = 0xffff8a451969fc00, gfp_mask = 0x220 } crash> common_audit_data ffff8a45ca717690 struct common_audit_data { type = 0x2, u = { path = { mnt = 0xffff8a45ca7176b0, dentry = 0xffff8a4500000000 }, dentry = 0xffff8a45ca7176b0, inode = 0xffff8a45ca7176b0, net = 0xffff8a45ca7176b0, cap = 0xca7176b0, ipc_id = 0xca7176b0, tsk = 0xffff8a45ca7176b0, key_struct = { key = 0xca7176b0, key_desc = 0xffff8a4500000000 struct: page excluded: kernel virtual address: ffff8a4500000000 type: "gdb_readmem_callback" struct: page excluded: kernel virtual address: ffff8a4500000000 type: "gdb_readmem_callback" struct: page excluded: kernel virtual address: ffff8a4500000000 type: "gdb_readmem_callback" <error: Cannot access memory at address 0xffff8a4500000000> }, kmod_name = 0xffff8a45ca7176b0 "", op = 0xffff8a45ca7176b0, file = 0xffff8a45ca7176b0, ibpkey = 0xffff8a45ca7176b0, ibendport = 0xffff8a45ca7176b0 }, { selinux_audit_data = 0xffff8a45ca71758c } } crash> selinux_audit_data 0xffff8a45ca71758c struct selinux_audit_data { ssid = 0x1, tsid = 0x53494e47, tclass = 0x4e47, <---- requested = 0x2, audited = 0x2, denied = 0x2, result = 0x0 } crash> sock 0xffff8a33e29aa940 |grep sk_security sk_security = 0xffff8a44ecde2e20, crash> sk_security_struct.sclass 0xffff8a44ecde2e20, sclass = 0x4e47 <---- crash> sk_security_struct 0xffff8a44ecde2e20 struct sk_security_struct { nlbl_state = 1397313095, nlbl_secattr = 0x53494e4753494e47, sid = 0x53494e47, peer_sid = 0x53494e47, sclass = 0x4e47 } crash> rd 0xffff8a44ecde2e20 4 ffff8a44ecde2e20: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2e30: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS -
Looking for
GNISGNISGNISGNISin the memory, we find it starts fromffff8a44ecd6f090, which is akmalloc-4096:crash> kmem 0xffff8a44ecde2e20 CACHE OBJSIZE ALLOCATED TOTAL SLABS SSIZE NAME ffff8a237fc03c00 32 135371 171264 1338 4k kmalloc-32 SLAB MEMORY NODE TOTAL ALLOCATED FREE ffffe10e4bb37880 ffff8a44ecde2000 1 128 2 126 FREE / [ALLOCATED] [ffff8a44ecde2e20] PAGE PHYSICAL MAPPING INDEX CNT FLAGS ffffe10e4bb37880 22ecde2000 0 ffff8a44ecde2e00 1 6fffff00000080 slab crash> kmem ffffe10e4bb37880 | grep -e ffff8a44ecde2000 -e ffff8a44ecde2e20 -e ffff8a44ecde2fe0 ffffe10e4bb37880 ffff8a44ecde2000 1 128 2 126 [ffff8a44ecde2000] [ffff8a44ecde2e20] [ffff8a44ecde2fe0] crash> rd ffff8a44ecde2000 512 | head ffff8a44ecde2000: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2010: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2020: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2030: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2040: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2050: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2060: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2070: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2080: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS ffff8a44ecde2090: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS crash> kmem ffff8a44ecd6f090 CACHE OBJSIZE ALLOCATED TOTAL SLABS SSIZE NAME ffff8a237fc03300 4096 563 1096 137 32k kmalloc-4096 SLAB MEMORY NODE TOTAL ALLOCATED FREE ffffe10e4bb35a00 ffff8a44ecd68000 1 8 4 4 FREE / [ALLOCATED] [ffff8a44ecd6f000] PAGE PHYSICAL MAPPING INDEX CNT FLAGS ffffe10e4bb35bc0 22ecd6f000 0 0 0 6fffff00008000 tail crash> rd ffff8a44ecd6f000 512 | head ffff8a44ecd6f000: 0000000020e23900 0000000000f10b00 .9. ............ ffff8a44ecd6f010: 0000001b0039e3c4 0000000000000000 ..9............. ffff8a44ecd6f020: 0000000000000000 000bf1020000001b ................ ffff8a44ecd6f030: 0039e40000000000 0000001b000bf103 ......9......... ffff8a44ecd6f040: 000000000039e5a4 0000001b000bf104 ..9............. ffff8a44ecd6f050: 0000000000000000 0039e74800000000 ............H.9. ffff8a44ecd6f060: 000bf1050000001b 000000000039e8ec ..........9..... ffff8a44ecd6f070: 00000000000bf106 0000001b0039ea18 ..........9..... ffff8a44ecd6f080: 0000000000000000 0000000000000000 ................ ffff8a44ecd6f090: 53494e4753494e47 53494e4753494e47 GNISGNISGNISGNIS crash> search 53494e4753494e47 -s ffff8a44ecd6f090 | cat -n | tail -10 212691 ffff8a44ecf20720: 53494e4753494e47 212692 ffff8a44ecf20728: 53494e4753494e47 212693 ffff8a44ecf20730: 53494e4753494e47 212694 ffff8a44ecf20738: 53494e4753494e47 212695 ffff8a44ecf20740: 53494e4753494e47 212696 ffff8a44ecf20748: 53494e4753494e47 212697 ffff8a44ecf20750: 53494e4753494e47 212698 ffff8a44ecf20758: 53494e4753494e47 212699 ffff8a44ecf20760: 53494e4753494e47 212700 ffff8a44ecf20768: 53494e4753494e47 crash> rd ffff8a44ecf20768 5 ffff8a44ecf20768: 53494e4753494e47 0000000053494e47 GNISGNISGNIS.... ffff8a44ecf20778: 0000000000000000 0000000000000000 ................ ffff8a44ecf20788: 0000000000000000 ........ -
The address of the kmalloc-4096 (ffff8a44ecd6f000) is in the stack of the pid on CPU 21:
crash> bt -c 21 PID: 3684 TASK: ffff8a45cdaf8000 CPU: 21 COMMAND: "vxiod" #0 [ffff8a45f7348e48] crash_nmi_callback at ffffffffa1858597 #1 [ffff8a45f7348e58] nmi_handle at ffffffffa1f9193c #2 [ffff8a45f7348eb0] do_nmi at ffffffffa1f91b5d #3 [ffff8a45f7348ef0] end_repeat_nmi at ffffffffa1f90d9c [exception RIP: voliomem_next_segment+0x11] RIP: ffffffffc2f4f5e1 RSP: ffff8a45cdb03ce8 RFLAGS: 00000202 RAX: 0000000053494e47 RBX: ffff8a44ecf20774 RCX: 0000000000004e00 RDX: 0000000017e20c00 RSI: ffff8a45f5128900 RDI: ffff8a45cdb03cf8 RBP: ffff8a45cdb03ce8 R8: 0000000000000000 R9: 0000000000000000 R10: ffff8a44f37eb400 R11: ffff8a45b9050c00 R12: ffff8a44ed06b49c R13: ffff8a44f37eb400 R14: 0000000039e3c400 R15: ffff8a44f37eb400 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- <NMI exception stack> --- #4 [ffff8a45cdb03ce8] voliomem_next_segment at ffffffffc2f4f5e1 [vxio] #5 [ffff8a45cdb03cf0] vol_ru_verification_data_unpack at ffffffffc30c7acc [vxio] #6 [ffff8a45cdb03d40] vol_ru_verify at ffffffffc3078f3a [vxio] #7 [ffff8a45cdb03da0] volrv_seclog_bulk_cleanup_verification at ffffffffc306ec85 [vxio] #8 [ffff8a45cdb03e18] volrv_seclog_write1_done at ffffffffc306ed91 [vxio] #9 [ffff8a45cdb03e48] voliod_iohandle at ffffffffc2ea7498 [vxio] #10 [ffff8a45cdb03e88] voliod_loop at ffffffffc2ea768c [vxio] #11 [ffff8a45cdb03ec8] kthread at ffffffffa18c5e61 crash> crash> bt -c 21 -f | grep ffff8a44ecd6f000 ffff8a45cdb03da8: ffff8a33f5569890 ffff8a44ecd6f000 The location of the address in stack: #7 [ffff8a45cdb03da0] volrv_seclog_bulk_cleanup_verification at ffffffffc306ec85 [vxio] -60= ffff8a45cdb03da8: ffff8a33f5569890 ffff8a44ecd6f000 <--- ^^^^^^^^^^^^^^^^ ffff8a45cdb03db8: ffff8a3387b00000 0000000000008000 ffff8a45cdb03dc8: 0000000000008000 ffff8a45f5128ee0 ffff8a45cdb03dd8: ffff8a33f5569890 00000000972fc42b rbx r12 ffff8a45cdb03de8: ffff8a33e532b400 ffff8a33f5657548 r13 r14 ffff8a45cdb03df8: 0000000000000000 ffff8a43c1f70000 r15 rbp ffff8a45cdb03e08: ffff8a33f5657400 ffff8a45cdb03e40 ffff8a45cdb03e18: ffffffffc306ed91 #8 [ffff8a45cdb03e18] volrv_seclog_write1_done at ffffffffc306ed91 [vxio] -
The 3rd party code
volrv_seclog_bulk_cleanup_verificationcallsvol_zalloc()to get/create the addressffff8a44ecd6f000and passes it tovol_ru_verification_data_unpack()which started looping writingGNISto the address:xffffffffc306ebf3 <volrv_seclog_bulk_cleanup_verification+0x53>: mov $0xd0,%esi 0xffffffffc306ebf8 <volrv_seclog_bulk_cleanup_verification+0x58>: call 0xffffffffc2f572c0 <vol_zalloc> 0xffffffffc306ebfd <volrv_seclog_bulk_cleanup_verification+0x5d>: mov $0xd0,%esi 0xffffffffc306ec02 <volrv_seclog_bulk_cleanup_verification+0x62>: mov $0x188,%edi 0xffffffffc306ec07 <volrv_seclog_bulk_cleanup_verification+0x67>: and $0xfffff000,%r14d 0xffffffffc306ec0e <volrv_seclog_bulk_cleanup_verification+0x6e>: mov %rax,-0x60(%rbp) <----
This solution is part of Red Hat’s fast-track publication program, providing a huge library of solutions that Red Hat engineers have created while supporting our customers. To give you the knowledge you need the instant it becomes available, these articles may be presented in a raw and unedited form.
Comments