The kernel crashes due to a hard lockup. Looks like some data race happens when installing an out-of-tree/unsigned module/driver namely fl2000 that We Red Hat do not maintain

Issue

• The kernel crashes due to a hard lockup. Looks like some data race happens when installing an out-of-tree/unsigned module/driver namely fl2000 that We Red Hat do not maintain.

[  542.146784] fl2000_device_probe:bInterfaceNumber:1 bAlternateSetting:0
[  542.146788] ------------[ cut here ]------------
[  542.146795] WARNING: CPU: 4 PID: 9179 at include/linux/kref.h:52 fl2000_device_probe+0x304/0x3e0 [fl2000]
[  542.146804] Modules linked in: fl2000(OE+) tcp_lp nvidia_uvm(OE) veth lp parport xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6_tables overlay(T) nf_conntrack_ftp ipt_REJECT nf_reject_ipv4 xt_conntrack iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) vfat fat ext4 mbcache jbd2 skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel iTCO_wdt iTCO_vendor_support kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg mei_me mei i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter tpm_crb binfmt_misc
[  542.146854]  ip_tables xfs libcrc32c sd_mod crc_t10dif crct10dif_generic cdc_ether usbnet mii mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci i40e igb libahci crct10dif_pclmul crct10dif_common nvme crc32c_intel libata nvme_core dca ptp pps_core i2c_algo_bit drm_panel_orientation_quirks wmi nfit libnvdimm uas usb_storage dm_mirror dm_region_hash dm_log dm_mod fuse
[  542.146882] CPU: 4 PID: 9179 Comm: insmod Kdump: loaded Tainted: P           OE  ------------ T 3.10.0-1160.66.1.el7.x86_64 #1
[  542.146884] Hardware name: Lenovo ThinkSystem SE350 -[7D1XCTO1WW]-/-[7D1XCTO1WW]-, BIOS -[HYE122F-1.19]- 04/27/2022
[  542.146885] Call Trace:
[  542.146892]  [<ffffffffa19865a9>] dump_stack+0x19/0x1b
[  542.146899]  [<ffffffffa129b408>] __warn+0xd8/0x100
[  542.146903]  [<ffffffffa129b54d>] warn_slowpath_null+0x1d/0x20
[  542.146909]  [<ffffffffc32d2504>] fl2000_device_probe+0x304/0x3e0 [fl2000]
[  542.146914]  [<ffffffffa17199ed>] usb_probe_interface+0x11d/0x320
[  542.146919]  [<ffffffffa16bbe85>] driver_probe_device+0xc5/0x3e0
[  542.146922]  [<ffffffffa16bc283>] __driver_attach+0x93/0xa0
[  542.146926]  [<ffffffffa16bc1f0>] ? __device_attach+0x50/0x50
[  542.146929]  [<ffffffffa16b9a25>] bus_for_each_dev+0x75/0xc0
[  542.146932]  [<ffffffffa16bb7fe>] driver_attach+0x1e/0x20
[  542.146935]  [<ffffffffa16bb2a0>] bus_add_driver+0x200/0x2d0
[  542.146938]  [<ffffffffa16bc914>] driver_register+0x64/0xf0
[  542.146941]  [<ffffffffa1718052>] usb_register_driver+0x82/0x160
[  542.146946]  [<ffffffffc32f1000>] ? 0xffffffffc32f0fff
[  542.146951]  [<ffffffffc32f101e>] init_module+0x1e/0x1000 [fl2000]
[  542.146958]  [<ffffffffa120210a>] do_one_initcall+0xba/0x240
[  542.146963]  [<ffffffffa131ea5a>] load_module+0x271a/0x2bb0
[  542.146967]  [<ffffffffa15b4ce0>] ? ddebug_proc_write+0x100/0x100
[  542.146972]  [<ffffffffa1450b83>] ? fput+0x13/0x20
[  542.146975]  [<ffffffffa131a5e3>] ? copy_module_from_fd.isra.44+0x53/0x150
[  542.146979]  [<ffffffffa131f0d6>] SyS_finit_module+0xa6/0xd0
[  542.146982]  [<ffffffffa1999f92>] system_call_fastpath+0x25/0x2a
[  542.146985] ---[ end trace d17c927d23901180 ]---
[  542.146986] fl2000_device_probe:streaming interface detected
[  542.147080] fl2000_device_probe:usb_dev(ffff921e43dcb000), minor_num(128), dev_name(fl2000-0) created
[  543.795014] fl2000_monitor_plugin_handler:Notify system to add monitor.
[  543.795049] fl2000_device_probe:bInterfaceNumber:2 bAlternateSetting:0
[  543.795053] fl2000_device_probe:Detect interrupt interface.
[  543.795056] fl2000_intr_pipe_create:found ep_num_intr_in(3)
[  543.795298] usbcore: registered new interface driver fl2000
[  556.585970] warning: `turbostat' uses 32-bit capabilities (legacy support in use)
[  597.824708] NMI watchdog: Watchdog detected hard LOCKUP on cpu 4
[  597.824711] Modules linked in:
[  597.824714]  fl2000(OE) tcp_lp nvidia_uvm(OE) veth lp parport xt_nat nf_conntrack_netlink xt_addrtype br_netfilter xt_CHECKSUM ipt_MASQUERADE nf_nat_masquerade_ipv4 tun bridge stp llc ebtable_filter ebtables devlink ip6_tables overlay(T) nf_conntrack_ftp ipt_REJECT nf_reject_ipv4 xt_conntrack iptable_filter iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack iptable_mangle sunrpc nvidia_drm(POE) nvidia_modeset(POE) nvidia(POE) vfat fat ext4 mbcache jbd2 skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel iTCO_wdt iTCO_vendor_support kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr sg mei_me mei i2c_i801 ipmi_si ipmi_devintf ipmi_msghandler acpi_pad acpi_power_meter tpm_crb binfmt_misc ip_tables xfs
[  597.824754]  libcrc32c sd_mod crc_t10dif crct10dif_generic cdc_ether usbnet mii mgag200 drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm ahci i40e igb libahci crct10dif_pclmul crct10dif_common nvme crc32c_intel libata nvme_core dca ptp pps_core i2c_algo_bit drm_panel_orientation_quirks wmi nfit libnvdimm uas usb_storage dm_mirror dm_region_hash dm_log dm_mod fuse
[  597.824777] CPU: 4 PID: 452 Comm: kworker/u144:17 Kdump: loaded Tainted: P        W  OE  ------------ T 3.10.0-1160.66.1.el7.x86_64 #1
[  597.824779] Hardware name: Lenovo ThinkSystem SE350 -[7D1XCTO1WW]-/-[7D1XCTO1WW]-, BIOS -[HYE122F-1.19]- 04/27/2022
[  597.824785] Workqueue: scsi_tmf_0 scmd_eh_abort_handler
[  597.824787] task: ffff9220d69a6300 ti: ffff9220d5044000 task.ti: ffff9220d5044000
[  597.824788] RIP: 0010:[<ffffffffa1317bd2>]  [<ffffffffa1317bd2>] native_queued_spin_lock_slowpath+0x122/0x200
[  597.824794] RSP: 0018:ffff9220d5047d50  EFLAGS: 00000046
[  597.824795] RAX: 0000000000000000 RBX: ffff921a2b8b88b8 RCX: 0000000000210000
[  597.824797] RDX: ffffffffa1f898d0 RSI: 000000002b8b8928 RDI: ffff921a2b8b88b8
[  597.824798] RBP: ffff9220d5047d50 R08: ffff9220de91b8c0 R09: 0000000000000000
[  597.824799] R10: b77dbbd8ccf1ea28 R11: 0000000000000293 R12: ffff92207591ea00
[  597.824801] R13: ffff921a2b8b88b0 R14: ffff9220d81b9b00 R15: 0000000000001200
[  597.824803] FS:  0000000000000000(0000) GS:ffff9220de900000(0000) knlGS:0000000000000000
[  597.824804] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  597.824806] CR2: 00007f1d7e4d5f80 CR3: 000000081e4ba000 CR4: 00000000007607e0
[  597.824807] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  597.824809] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  597.824810] PKRU: 00000000
[  597.824811] Call Trace:
[  597.824817]  [<ffffffffa1980d63>] queued_spin_lock_slowpath+0xb/0xf
[  597.824822]  [<ffffffffa198ec08>] _raw_spin_lock_irq+0x28/0x30
[  597.824825]  [<ffffffffa198c6c7>] wait_for_completion+0x37/0x140
[  597.824829]  [<ffffffffa12e6451>] ? put_prev_entity+0x31/0x400
[  597.824833]  [<ffffffffa122b59e>] ? __switch_to+0xce/0x580
[  597.824840]  [<ffffffffc02e3de0>] command_abort+0x80/0xa0 [usb_storage]
[  597.824842]  [<ffffffffa16e9446>] scmd_eh_abort_handler+0xc6/0x2a0
[  597.824846]  [<ffffffffa12bdfbf>] process_one_work+0x17f/0x440
[  597.824849]  [<ffffffffa12bf0d6>] worker_thread+0x126/0x3c0
[  597.824852]  [<ffffffffa12befb0>] ? manage_workers.isra.26+0x2a0/0x2a0
[  597.824855]  [<ffffffffa12c5f91>] kthread+0xd1/0xe0
[  597.824857]  [<ffffffffa12c5ec0>] ? insert_kthread_work+0x40/0x40
[  597.824862]  [<ffffffffa1999df7>] ret_from_fork_nospec_begin+0x21/0x21
[  597.824865]  [<ffffffffa12c5ec0>] ? insert_kthread_work+0x40/0x40
[  597.824866] Code: 13 48 c1 ea 0d 48 98 83 e2 30 48 81 c2 c0 b8 01 00 48 03 14 c5 60 18 f5 a1 4c 89 02 41 8b 40 08 85 c0 75 0f 0f 1f 44 00 00 f3 90 <41> 8b 40 08 85 c0 74 f6 4d 8b 08 4d 85 c9 74 04 41 0f 18 09 8b 
[  597.824890] Kernel panic - not syncing: Hard LOCKUP
[  597.824910] CPU: 4 PID: 452 Comm: kworker/u144:17 Kdump: loaded Tainted: P        W  OE  ------------ T 3.10.0-1160.66.1.el7.x86_64 #1
[  597.824943] Hardware name: Lenovo ThinkSystem SE350 -[7D1XCTO1WW]-/-[7D1XCTO1WW]-, BIOS -[HYE122F-1.19]- 04/27/2022
[  597.824970] Workqueue: scsi_tmf_0 scmd_eh_abort_handler
[  597.824986] Call Trace:
[  597.824994]  <NMI>  [<ffffffffa19865a9>] dump_stack+0x19/0x1b
[  597.825013]  [<ffffffffa19802b1>] panic+0xe8/0x21f
[  597.825029]  [<ffffffffa122f958>] ? show_regs+0x58/0x290
[  597.825772]  [<ffffffffa129b0ef>] nmi_panic+0x3f/0x40
[  597.826507]  [<ffffffffa134f319>] watchdog_overflow_callback+0x119/0x140
[  597.827248]  [<ffffffffa13a8de7>] __perf_event_overflow+0x57/0x100
[  597.827977]  [<ffffffffa13b25e4>] perf_event_overflow+0x14/0x20
[  597.828696]  [<ffffffffa120a9b0>] handle_pmi_common+0x1a0/0x250
[  597.829415]  [<ffffffffa1588da8>] ? ioremap_page_range+0x2e8/0x480
[  597.830128]  [<ffffffffa1405b84>] ? vunmap_page_range+0x234/0x470
[  597.830825]  [<ffffffffa16502c6>] ? ghes_copy_tofrom_phys+0x116/0x210
[  597.831519]  [<ffffffffa120ac8f>] intel_pmu_handle_irq+0xcf/0x1d0
[  597.832212]  [<ffffffffa1990031>] perf_event_nmi_handler+0x31/0x50
[  597.832894]  [<ffffffffa199193c>] nmi_handle.isra.0+0x8c/0x150
[  597.833569]  [<ffffffffa1991c18>] do_nmi+0x218/0x460
[  597.834240]  [<ffffffffa1990d9c>] end_repeat_nmi+0x1e/0x81
[  597.834908]  [<ffffffffa1317bd2>] ? native_queued_spin_lock_slowpath+0x122/0x200
[  597.835574]  [<ffffffffa1317bd2>] ? native_queued_spin_lock_slowpath+0x122/0x200
[  597.836232]  [<ffffffffa1317bd2>] ? native_queued_spin_lock_slowpath+0x122/0x200
[  597.836879]  <EOE>  [<ffffffffa1980d63>] queued_spin_lock_slowpath+0xb/0xf
[  597.837534]  [<ffffffffa198ec08>] _raw_spin_lock_irq+0x28/0x30
[  597.838180]  [<ffffffffa198c6c7>] wait_for_completion+0x37/0x140
[  597.838820]  [<ffffffffa12e6451>] ? put_prev_entity+0x31/0x400
[  597.839460]  [<ffffffffa122b59e>] ? __switch_to+0xce/0x580
[  597.840105]  [<ffffffffc02e3de0>] command_abort+0x80/0xa0 [usb_storage]
[  597.840736]  [<ffffffffa16e9446>] scmd_eh_abort_handler+0xc6/0x2a0
[  597.841364]  [<ffffffffa12bdfbf>] process_one_work+0x17f/0x440
[  597.841997]  [<ffffffffa12bf0d6>] worker_thread+0x126/0x3c0
[  597.842625]  [<ffffffffa12befb0>] ? manage_workers.isra.26+0x2a0/0x2a0
[  597.843248]  [<ffffffffa12c5f91>] kthread+0xd1/0xe0
[  597.843860]  [<ffffffffa12c5ec0>] ? insert_kthread_work+0x40/0x40
[  597.844471]  [<ffffffffa1999df7>] ret_from_fork_nospec_begin+0x21/0x21
[  597.845078]  [<ffffffffa12c5ec0>] ? insert_kthread_work+0x40/0x40