Kernel BUG during processing pipelined response and a lot of possible problems in the appropriate code #2545
Description
Describe the issue
Detailed description of the issue.
Expected Behavior
Describe what did you expect.
To Reproduce
I reproduced it on the t_stress.test_ddos but it was reproduced on the specific Tempesta FW branch and old test version. We should write tests which can reproduce this BUG and possible problems in this part of Tempesta FW code.
Version or commit hash
This problem was reproduced on the #2456 PR, but during investigation I can say that we have the same problem on 0.8 and master branch.
Stacktrace or debug log
[ 367.042412] WARNING: CPU: 3 PID: 38 at /home/evgeny/workdir/tempesta/fw/http.c:1003 tfw_http_conn_msg_free+0xae/0x2a0 [tempesta_fw]
[ 367.043832] Modules linked in: tcp_diag inet_diag tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) vhost_vsock vmw_vsock_virtio_transport_common vhost vhost_iotlb vsock xt_conntrack xt_MASQUERADE xt_set ip_set xt_addrtype nft_compat xfrm_user xfrm_algo nft_masq nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 bridge stp llc nf_tables nfnetlink overlay intel_rapl_msr intel_rapl_common snd_hda_codec_generic intel_uncore_frequency_common intel_pmc_core snd_hda_intel intel_vsec pmt_telemetry snd_intel_dspcfg pmt_class snd_intel_sdw_acpi snd_hda_codec snd_hda_core kvm_intel snd_hwdep snd_pcm kvm binfmt_misc snd_seq_midi snd_seq_midi_event rapl snd_rawmidi joydev snd_seq 9pnet_virtio 9pnet nls_iso8859_1 snd_seq_device input_leds snd_timer i2c_i801 netfs serio_raw snd i2c_mux lpc_ich qxl soundcore i2c_smbus drm_ttm_helper qemu_fw_cfg mac_hid ttm sch_fq_codel msr parport_pc ppdev lp parport efi_pstore ip_tables x_tables autofs4 btrfs xor raid6_pq libcrc32c psmouse ahci virtio_rng libahci
[ 367.044034] hid_generic usbhid hid
[ 367.051417] CPU: 3 UID: 0 PID: 38 Comm: ksoftirqd/3 Tainted: G OE 6.12.12+ #94
[ 367.052117] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 367.052584] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.2-debian-1.16.2-1 04/01/2014
[ 367.053340] RIP: 0010:tfw_http_conn_msg_free+0xae/0x2a0 [tempesta_fw]
[ 367.053916] Code: 00 00 00 00 00 fc ff df 48 8d 7b 38 48 89 fa 48 c1 ea 03 80 3c 02 00 0f 85 a2 01 00 00 48 8b 4b 38 48 85 c9 0f 84 8d 00 00 00 <0f> 0b 48 8d b9 c8 01 00 00 65 44 8b 0d 6d e9 f6 3d 48 b8 00 00 00
[ 367.055414] RSP: 0018:ffff888100d169e0 EFLAGS: 00010282
[ 367.056058] RAX: dffffc0000000000 RBX: ffff88805cf0a020 RCX: ffff8881c48e9020
[ 367.056730] RDX: 1ffff1100b9e140b RSI: 0000000000000000 RDI: ffff88805cf0a058
[ 367.057367] RBP: ffff888100d16a10 R08: 0000000000000000 R09: 0000000000000000
[ 367.057947] R10: 0000000000000000 R11: 0000000000000000 R12: ffff888044f4d870
[ 367.058536] R13: ffff88805cf0a0e8 R14: ffff88803010ba68 R15: ffff88806175a258
[ 367.059118] FS: 0000000000000000(0000) GS:ffff888230380000(0000) knlGS:0000000000000000
[ 367.059781] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 367.060267] CR2: 000075d9d460d210 CR3: 0000000104ca8000 CR4: 0000000000752ef0
[ 367.060871] PKRU: 55555554
[ 367.061105] Call Trace:
[ 367.061330]
[ 367.061520] ? show_regs+0x6c/0x80
[ 367.061819] ? __warn+0xd1/0x270
[ 367.062296] ? tfw_http_conn_msg_free+0xae/0x2a0 [tempesta_fw]
[ 367.062826] ? report_bug+0x282/0x2f0
[ 367.063159] ? handle_bug+0x6e/0xc0
[ 367.063481] ? exc_invalid_op+0x18/0x50
[ 367.063805] ? asm_exc_invalid_op+0x1b/0x20
[ 367.064165] ? tfw_http_conn_msg_free+0xae/0x2a0 [tempesta_fw]
[ 367.064715] tfw_http_req_evict_dropped+0x406/0x550 [tempesta_fw]
[ 367.065268] tfw_http_conn_resend+0x221/0xf70 [tempesta_fw]
[ 367.065792] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.066331] tfw_http_popreq+0xf7c/0x17d0 [tempesta_fw]
[ 367.066829] ? __pfx_tfw_http_popreq+0x10/0x10 [tempesta_fw]
[ 367.067373] ? __kasan_check_write+0x14/0x30
[ 367.067830] ? do_raw_spin_lock+0x134/0x290
[ 367.068208] tfw_http_resp_cache+0x1dd/0x7e0 [tempesta_fw]
[ 367.069236] ? do_raw_spin_unlock+0x14b/0x200
[ 367.069615] ? __pfx_tfw_http_resp_cache+0x10/0x10 [tempesta_fw]
[ 367.070162] ? __kasan_check_read+0x11/0x20
[ 367.070527] ? tfw_http_conn_msg_alloc+0x31e/0x8e0 [tempesta_fw]
[ 367.071111] tfw_http_msg_process_generic+0x8fd/0x1ad0 [tempesta_fw]
[ 367.071708] ? smpboot_thread_fn+0x289/0x660
[ 367.072196] ? kthread+0x29e/0x3a0
[ 367.072592] ? ret_from_fork+0x44/0x90
[ 367.072909] ? ret_from_fork_asm+0x1a/0x30
[ 367.073266] ? __pfx_tfw_http_msg_process_generic+0x10/0x10 [tempesta_fw]
[ 367.073865] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.074298] ? __pfx_free_object_rcu+0x10/0x10
[ 367.074688] ? call_rcu+0x34/0x50
[ 367.075079] ? __kasan_slab_free+0x5d/0x80
[ 367.075434] ? kmem_cache_free+0x18a/0x560
[ 367.075973] ? __kasan_check_write+0x14/0x30
[ 367.076339] tfw_http_msg_process+0xc4/0x160 [tempesta_fw]
[ 367.076842] tfw_connection_recv+0x18a/0x420 [tempesta_fw]
[ 367.077341] ? consume_skb+0xb6/0x1f0
[ 367.077659] ? __pfx_tfw_connection_recv+0x10/0x10 [tempesta_fw]
[ 367.078236] ss_tcp_process_data+0x656/0x13b0 [tempesta_fw]
[ 367.078767] ? __pfx_ss_tcp_process_data+0x10/0x10 [tempesta_fw]
[ 367.079316] ? __sk_mem_schedule+0x78/0x100
[ 367.079667] ss_tcp_state_change+0x19b/0x510 [tempesta_fw]
[ 367.080228] tcp_fin+0x234/0x590
[ 367.080527] tcp_data_queue+0x1e5b/0x5fe0
[ 367.080889] ? __pfx_selinux_socket_sock_rcv_skb+0x10/0x10
[ 367.081524] ? __pfx_tcp_data_queue+0x10/0x10
[ 367.081889] ? __pfx_tcp_urg+0x10/0x10
[ 367.082238] ? kvm_clock_get_cycles+0x18/0x40
[ 367.082675] ? ktime_get+0x64/0x160
[ 367.083047] tcp_rcv_established+0x6a9/0x20f0
[ 367.083419] ? security_sock_rcv_skb+0x77/0x190
[ 367.083986] ? sk_filter_trim_cap+0x3ae/0x7e0
[ 367.084374] ? __pfx_tcp_rcv_established+0x10/0x10
[ 367.084789] ? do_raw_spin_lock+0x134/0x290
[ 367.085168] ? __kasan_check_read+0x11/0x20
[ 367.085545] tcp_v4_do_rcv+0x55a/0x8e0
[ 367.085863] tcp_v4_rcv+0x2d6f/0x47e0
[ 367.086172] ? __pfx_tcp_v4_rcv+0x10/0x10
[ 367.086517] ? raw_local_deliver+0x337/0xb30
[ 367.086879] ? __kasan_check_read+0x11/0x20
[ 367.087236] ? __pfx_raw_local_deliver+0x10/0x10
[ 367.087624] ? __pfx_nf_confirm+0x10/0x10 [nf_conntrack]
[ 367.088078] ? __kasan_check_read+0x11/0x20
[ 367.088437] ip_protocol_deliver_rcu+0x67/0x390
[ 367.088832] ip_local_deliver_finish+0x283/0x370
[ 367.089280] ip_local_deliver+0x1ae/0x390
[ 367.089616] ? __pfx_ip_local_deliver+0x10/0x10
[ 367.089995] ? __pfx_ip_local_deliver_finish+0x10/0x10
[ 367.090483] ? ip_rcv_finish_core.isra.0+0x12f5/0x18a0
[ 367.090910] ? nf_hook_slow+0xaa/0x200
[ 367.091236] ip_rcv+0x2fc/0x380
[ 367.091505] ? __pfx_ip_rcv+0x10/0x10
[ 367.091813] ? __pfx_ip_rcv_finish+0x10/0x10
[ 367.092172] ? netif_receive_skb_list_internal+0x5f2/0xd50
[ 367.092643] ? __pfx_ip_rcv+0x10/0x10
[ 367.092951] __netif_receive_skb_core.constprop.0+0xa4e/0x3f50
[ 367.093436] ? virtqueue_add_inbuf_ctx+0x120/0x160
[ 367.093841] ? __pfx_virtqueue_add_inbuf_ctx+0x10/0x10
[ 367.094272] ? sg_init_one+0x28/0x1a0
[ 367.094761] ? __pfx___netif_receive_skb_core.constprop.0+0x10/0x10
[ 367.095345] ? page_frag_free+0xba/0x250
[ 367.095823] ? skb_free_head+0x122/0x280
[ 367.096153] ? __kasan_check_write+0x14/0x30
[ 367.096535] ? __free_old_xmit+0x150/0x630
[ 367.097022] ? __kasan_check_write+0x14/0x30
[ 367.097442] ? __free_old_xmit+0x356/0x630
[ 367.097781] __netif_receive_skb_one_core+0xb2/0x1d0
[ 367.098200] ? __pfx___netif_receive_skb_one_core+0x10/0x10
[ 367.098659] ? __kasan_check_write+0x14/0x30
[ 367.099114] ? do_raw_spin_lock+0x134/0x290
[ 367.099553] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.099931] __netif_receive_skb+0x23/0x120
[ 367.100289] process_backlog+0x1a1/0x5c0
[ 367.100629] __napi_poll+0xa7/0x540
[ 367.100926] net_rx_action+0x560/0xfe0
[ 367.101249] ? __pfx_net_rx_action+0x10/0x10
[ 367.101670] ? __pfx___run_timers+0x10/0x10
[ 367.102219] ? __kasan_check_write+0x14/0x30
[ 367.102590] ? do_raw_spin_lock+0x134/0x290
[ 367.102958] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.103353] handle_softirqs+0x171/0x590
[ 367.103775] ? __pfx_run_ksoftirqd+0x10/0x10
[ 367.104175] ? __pfx_run_ksoftirqd+0x10/0x10
[ 367.104566] run_ksoftirqd+0x3a/0x60
[ 367.104958] smpboot_thread_fn+0x289/0x660
[ 367.105408] ? __pfx_smpboot_thread_fn+0x10/0x10
[ 367.105829] kthread+0x29e/0x3a0
[ 367.106128] ? __pfx_kthread+0x10/0x10
[ 367.106467] ret_from_fork+0x44/0x90
[ 367.106847] ? __pfx_kthread+0x10/0x10
[ 367.107200] ret_from_fork_asm+0x1a/0x30
[ 367.107545]
[ 367.107763] ---[ end trace 0000000000000000 ]---
[ 367.115663] RIP: 0010:tfw_http_msg_process_generic+0x365/0x1ad0 [tempesta_fw]
[ 367.116607] Code: ff ff 4c 89 bd 60 fe ff ff 4d 89 f7 4d 8d b7 c8 01 00 00 48 85 d2 0f 84 74 02 00 00 48 8d ba c8 00 00 00 48 89 f8 48 c1 e8 03 <42> 80 3c 28 00 0f 85 0c 11 00 00 48 8b 82 c8 00 00 00 48 89 85 88
[ 367.118405] RSP: 0018:ffff888100d16cf8 EFLAGS: 00010202
[ 367.118987] RAX: 0000000000000019 RBX: ffff888100d16eb0 RCX: 0000000000000000
[ 367.119585] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 00000000000000c8
[ 367.120214] RBP: ffff888100d16ed8 R08: 0000000000000000 R09: 0000000000000000
[ 367.120847] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8881c1985380
[ 367.121483] R13: dffffc0000000000 R14: ffff8881c48e91e8 R15: ffff8881c48e9020
[ 367.122184] FS: 0000000000000000(0000) GS:ffff888230380000(0000) knlGS:0000000000000000
[ 367.122865] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 367.123336] CR2: 000075d9d460d210 CR3: 00000002558a6000 CR4: 0000000000752ef0
[ 367.123923] PKRU: 55555554
[ 367.124158] Call Trace:
[ 367.124386]
[ 367.124592] ? show_regs+0x6c/0x80
[ 367.124906] ? die_addr+0x41/0xc0
[ 367.125208] ? exc_general_protection+0x158/0x250
[ 367.125600] ? asm_exc_general_protection+0x27/0x30
[ 367.126012] ? tfw_http_msg_process_generic+0x365/0x1ad0 [tempesta_fw]
[ 367.126599] ? tfw_http_msg_process_generic+0x606/0x1ad0 [tempesta_fw]
[ 367.127189] ? smpboot_thread_fn+0x289/0x660
[ 367.127614] ? kthread+0x29e/0x3a0
[ 367.127902] ? ret_from_fork+0x44/0x90
[ 367.128215] ? ret_from_fork_asm+0x1a/0x30
[ 367.128846] ? __pfx_tfw_http_msg_process_generic+0x10/0x10 [tempesta_fw]
[ 367.129508] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.129948] ? __pfx_free_object_rcu+0x10/0x10
[ 367.130330] ? call_rcu+0x34/0x50
[ 367.130620] ? __kasan_slab_free+0x5d/0x80
[ 367.130968] ? kmem_cache_free+0x18a/0x560
[ 367.131332] ? __kasan_check_write+0x14/0x30
[ 367.131714] tfw_http_msg_process+0xc4/0x160 [tempesta_fw]
[ 367.132223] tfw_connection_recv+0x18a/0x420 [tempesta_fw]
[ 367.132737] ? consume_skb+0xb6/0x1f0
[ 367.133048] ? __pfx_tfw_connection_recv+0x10/0x10 [tempesta_fw]
[ 367.133587] ss_tcp_process_data+0x656/0x13b0 [tempesta_fw]
[ 367.134098] ? __pfx_ss_tcp_process_data+0x10/0x10 [tempesta_fw]
[ 367.134642] ? __sk_mem_schedule+0x78/0x100
[ 367.134997] ss_tcp_state_change+0x19b/0x510 [tempesta_fw]
[ 367.135506] tcp_fin+0x234/0x590
[ 367.135783] tcp_data_queue+0x1e5b/0x5fe0
[ 367.136118] ? __pfx_selinux_socket_sock_rcv_skb+0x10/0x10
[ 367.136586] ? __pfx_tcp_data_queue+0x10/0x10
[ 367.136955] ? __pfx_tcp_urg+0x10/0x10
[ 367.137420] ? kvm_clock_get_cycles+0x18/0x40
[ 367.137895] ? ktime_get+0x64/0x160
[ 367.138243] tcp_rcv_established+0x6a9/0x20f0
[ 367.138623] ? security_sock_rcv_skb+0x77/0x190
[ 367.139013] ? sk_filter_trim_cap+0x3ae/0x7e0
[ 367.139397] ? __pfx_tcp_rcv_established+0x10/0x10
[ 367.139799] ? do_raw_spin_lock+0x134/0x290
[ 367.140157] ? __kasan_check_read+0x11/0x20
[ 367.140514] tcp_v4_do_rcv+0x55a/0x8e0
[ 367.140878] tcp_v4_rcv+0x2d6f/0x47e0
[ 367.141290] ? __pfx_tcp_v4_rcv+0x10/0x10
[ 367.141640] ? raw_local_deliver+0x337/0xb30
[ 367.142012] ? __kasan_check_read+0x11/0x20
[ 367.143429] ? __pfx_raw_local_deliver+0x10/0x10
[ 367.144181] ? __pfx_nf_confirm+0x10/0x10 [nf_conntrack]
[ 367.145500] ? __kasan_check_read+0x11/0x20
[ 367.146163] ip_protocol_deliver_rcu+0x67/0x390
[ 367.147395] ip_local_deliver_finish+0x283/0x370
[ 367.148149] ip_local_deliver+0x1ae/0x390
[ 367.148808] ? __pfx_ip_local_deliver+0x10/0x10
[ 367.149506] ? __pfx_ip_local_deliver_finish+0x10/0x10
[ 367.150268] ? ip_rcv_finish_core.isra.0+0x12f5/0x18a0
[ 367.151718] ? nf_hook_slow+0xaa/0x200
[ 367.152349] ip_rcv+0x2fc/0x380
[ 367.153300] ? __pfx_ip_rcv+0x10/0x10
[ 367.153883] ? __pfx_ip_rcv_finish+0x10/0x10
[ 367.154555] ? netif_receive_skb_list_internal+0x5f2/0xd50
[ 367.155459] ? __pfx_ip_rcv+0x10/0x10
[ 367.155985] __netif_receive_skb_core.constprop.0+0xa4e/0x3f50
[ 367.156833] ? virtqueue_add_inbuf_ctx+0x120/0x160
[ 367.157501] ? __pfx_virtqueue_add_inbuf_ctx+0x10/0x10
[ 367.159074] ? sg_init_one+0x28/0x1a0
[ 367.159656] ? __pfx___netif_receive_skb_core.constprop.0+0x10/0x10
[ 367.160565] ? page_frag_free+0xba/0x250
[ 367.161141] ? skb_free_head+0x122/0x280
[ 367.161803] ? __kasan_check_write+0x14/0x30
[ 367.162412] ? __free_old_xmit+0x150/0x630
[ 367.163028] ? __kasan_check_write+0x14/0x30
[ 367.163635] ? __free_old_xmit+0x356/0x630
[ 367.164247] __netif_receive_skb_one_core+0xb2/0x1d0
[ 367.165035] ? __pfx___netif_receive_skb_one_core+0x10/0x10
[ 367.165871] ? __kasan_check_write+0x14/0x30
[ 367.166506] ? do_raw_spin_lock+0x134/0x290
[ 367.167096] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.168027] __netif_receive_skb+0x23/0x120
[ 367.168877] process_backlog+0x1a1/0x5c0
[ 367.169622] __napi_poll+0xa7/0x540
[ 367.170216] net_rx_action+0x560/0xfe0
[ 367.170864] ? __pfx_net_rx_action+0x10/0x10
[ 367.171636] ? __pfx___run_timers+0x10/0x10
[ 367.172434] ? __kasan_check_write+0x14/0x30
[ 367.173179] ? do_raw_spin_lock+0x134/0x290
[ 367.173973] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 367.175211] handle_softirqs+0x171/0x590
[ 367.175927] ? __pfx_run_ksoftirqd+0x10/0x10
[ 367.176603] ? __pfx_run_ksoftirqd+0x10/0x10
[ 367.177317] run_ksoftirqd+0x3a/0x60
[ 367.177883] smpboot_thread_fn+0x289/0x660
[ 367.178444] ? __pfx_smpboot_thread_fn+0x10/0x10
[ 367.179196] kthread+0x29e/0x3a0
[ 367.179732] ? __pfx_kthread+0x10/0x10
[ 367.180363] ret_from_fork+0x44/0x90
[ 367.181040] ? __pfx_kthread+0x10/0x10
[ 367.181665] ret_from_fork_asm+0x1a/0x30
[ 367.182274]
[ 367.182627] Modules linked in: tcp_diag inet_diag tempesta_fw(OE) tempesta_db(OE) tempesta_tls(OE) tempesta_lib(OE) vhost_vsock vmw_vsock_virtio_transport_common vhost vhost_iotlb vsock xt_conntrack xt_MASQUERADE xt_set ip_set xt_addrtype nft_compat xfrm_user xfrm_algo nft_masq nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 bridge stp llc nf_tables nfnetlink overlay intel_rapl_msr intel_rapl_common snd_hda_codec_generic intel_uncore_frequency_common intel_pmc_core snd_hda_intel intel_vsec pmt_telemetry snd_intel_dspcfg pmt_class snd_intel_sdw_acpi snd_hda_codec snd_hda_core kvm_intel snd_hwdep snd_pcm kvm binfmt_misc snd_seq_midi snd_seq_midi_event rapl snd_rawmidi joydev snd_seq 9pnet_virtio 9pnet nls_iso8859_1 snd_seq_device input_leds snd_timer i2c_i801 netfs serio_raw snd i2c_mux lpc_ich qxl soundcore i2c_smbus drm_ttm_helper qemu_fw_cfg mac_hid ttm sch_fq_codel msr parport_pc ppdev lp parport efi_pstore ip_tables x_tables autofs4 btrfs xor raid6_pq libcrc32c psmouse ahci virtio_rng libahci
[ 367.182875] hid_generic usbhid hid
[ 367.197053] ---[ end trace 0000000000000000 ]---
[ 367.197786] RIP: 0010:tfw_http_msg_process_generic+0x365/0x1ad0 [tempesta_fw]