mirror of
https://github.com/hardkernel/linux.git
synced 2026-03-28 13:30:29 +09:00
52da02521e
commit d9ccb18f83ea2bb654289b6ecf014fd267cc988b upstream.
Soft lockups have been observed on a cluster of Linux-based edge routers
located in a highly dynamic environment. Using the `bird` service, these
routers continuously update BGP-advertised routes due to frequently
changing nexthop destinations, while also managing significant IPv6
traffic. The lockups occur during the traversal of the multipath
circular linked-list in the `fib6_select_path` function, particularly
while iterating through the siblings in the list. The issue typically
arises when the nodes of the linked list are unexpectedly deleted
concurrently on a different core—indicated by their 'next' and
'previous' elements pointing back to the node itself and their reference
count dropping to zero. This results in an infinite loop, leading to a
soft lockup that triggers a system panic via the watchdog timer.
Apply RCU primitives in the problematic code sections to resolve the
issue. Where necessary, update the references to fib6_siblings to
annotate or use the RCU APIs.
Include a test script that reproduces the issue. The script
periodically updates the routing table while generating a heavy load
of outgoing IPv6 traffic through multiple iperf3 clients. It
consistently induces infinite soft lockups within a couple of minutes.
Kernel log:
0 [ffffbd13003e8d30] machine_kexec at ffffffff8ceaf3eb
1 [ffffbd13003e8d90] __crash_kexec at ffffffff8d0120e3
2 [ffffbd13003e8e58] panic at ffffffff8cef65d4
3 [ffffbd13003e8ed8] watchdog_timer_fn at ffffffff8d05cb03
4 [ffffbd13003e8f08] __hrtimer_run_queues at ffffffff8cfec62f
5 [ffffbd13003e8f70] hrtimer_interrupt at ffffffff8cfed756
6 [ffffbd13003e8fd0] __sysvec_apic_timer_interrupt at ffffffff8cea01af
7 [ffffbd13003e8ff0] sysvec_apic_timer_interrupt at ffffffff8df1b83d
-- <IRQ stack> --
8 [ffffbd13003d3708] asm_sysvec_apic_timer_interrupt at ffffffff8e000ecb
[exception RIP: fib6_select_path+299]
RIP: ffffffff8ddafe7b RSP: ffffbd13003d37b8 RFLAGS: 00000287
RAX: ffff975850b43600 RBX: ffff975850b40200 RCX: 0000000000000000
RDX: 000000003fffffff RSI: 0000000051d383e4 RDI: ffff975850b43618
RBP: ffffbd13003d3800 R8: 0000000000000000 R9: ffff975850b40200
R10: 0000000000000000 R11: 0000000000000000 R12: ffffbd13003d3830
R13: ffff975850b436a8 R14: ffff975850b43600 R15: 0000000000000007
ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018
9 [ffffbd13003d3808] ip6_pol_route at ffffffff8ddb030c
10 [ffffbd13003d3888] ip6_pol_route_input at ffffffff8ddb068c
11 [ffffbd13003d3898] fib6_rule_lookup at ffffffff8ddf02b5
12 [ffffbd13003d3928] ip6_route_input at ffffffff8ddb0f47
13 [ffffbd13003d3a18] ip6_rcv_finish_core.constprop.0 at ffffffff8dd950d0
14 [ffffbd13003d3a30] ip6_list_rcv_finish.constprop.0 at ffffffff8dd96274
15 [ffffbd13003d3a98] ip6_sublist_rcv at ffffffff8dd96474
16 [ffffbd13003d3af8] ipv6_list_rcv at ffffffff8dd96615
17 [ffffbd13003d3b60] __netif_receive_skb_list_core at ffffffff8dc16fec
18 [ffffbd13003d3be0] netif_receive_skb_list_internal at ffffffff8dc176b3
19 [ffffbd13003d3c50] napi_gro_receive at ffffffff8dc565b9
20 [ffffbd13003d3c80] ice_receive_skb at ffffffffc087e4f5 [ice]
21 [ffffbd13003d3c90] ice_clean_rx_irq at ffffffffc0881b80 [ice]
22 [ffffbd13003d3d20] ice_napi_poll at ffffffffc088232f [ice]
23 [ffffbd13003d3d80] __napi_poll at ffffffff8dc18000
24 [ffffbd13003d3db8] net_rx_action at ffffffff8dc18581
25 [ffffbd13003d3e40] __do_softirq at ffffffff8df352e9
26 [ffffbd13003d3eb0] run_ksoftirqd at ffffffff8ceffe47
27 [ffffbd13003d3ec0] smpboot_thread_fn at ffffffff8cf36a30
28 [ffffbd13003d3ee8] kthread at ffffffff8cf2b39f
29 [ffffbd13003d3f28] ret_from_fork at ffffffff8ce5fa64
30 [ffffbd13003d3f50] ret_from_fork_asm at ffffffff8ce03cbb
Fixes: 66f5d6ce53 ("ipv6: replace rwlock with rcu and spinlock in fib6_table")
Reported-by: Adrian Oliver <kernel@aoliver.ca>
Signed-off-by: Omid Ehtemam-Haghighi <omid.ehtemamhaghighi@menlosecurity.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Ido Schimmel <idosch@idosch.org>
Cc: Kuniyuki Iwashima <kuniyu@amazon.com>
Cc: Simon Horman <horms@kernel.org>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://patch.msgid.link/20241106010236.1239299-1-omid.ehtemamhaghighi@menlosecurity.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Rajani Kantha <rajanikantha@engineer.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
152 lines
5.7 KiB
Makefile
152 lines
5.7 KiB
Makefile
# SPDX-License-Identifier: GPL-2.0
|
|
# Makefile for net selftests
|
|
|
|
CFLAGS = -Wall -Wl,--no-as-needed -O2 -g
|
|
CFLAGS += -I../../../../usr/include/ $(KHDR_INCLUDES)
|
|
# Additional include paths needed by kselftest.h
|
|
CFLAGS += -I../
|
|
|
|
TEST_PROGS := run_netsocktests run_afpackettests test_bpf.sh netdevice.sh \
|
|
rtnetlink.sh xfrm_policy.sh test_blackhole_dev.sh
|
|
TEST_PROGS += fib_tests.sh fib-onlink-tests.sh pmtu.sh udpgso.sh ip_defrag.sh
|
|
TEST_PROGS += udpgso_bench.sh fib_rule_tests.sh msg_zerocopy.sh psock_snd.sh
|
|
TEST_PROGS += udpgro_bench.sh udpgro.sh test_vxlan_under_vrf.sh reuseport_addr_any.sh
|
|
TEST_PROGS += test_vxlan_fdb_changelink.sh so_txtime.sh ipv6_flowlabel.sh
|
|
TEST_PROGS += tcp_fastopen_backup_key.sh fcnal-test.sh l2tp.sh traceroute.sh
|
|
TEST_PROGS += fin_ack_lat.sh fib_nexthop_multiprefix.sh fib_nexthops.sh fib_nexthop_nongw.sh
|
|
TEST_PROGS += altnames.sh icmp.sh icmp_redirect.sh ip6_gre_headroom.sh
|
|
TEST_PROGS += route_localnet.sh
|
|
TEST_PROGS += reuseaddr_ports_exhausted.sh
|
|
TEST_PROGS += txtimestamp.sh
|
|
TEST_PROGS += vrf-xfrm-tests.sh
|
|
TEST_PROGS += rxtimestamp.sh
|
|
TEST_PROGS += devlink_port_split.py
|
|
TEST_PROGS += drop_monitor_tests.sh
|
|
TEST_PROGS += vrf_route_leaking.sh
|
|
TEST_PROGS += bareudp.sh
|
|
TEST_PROGS += amt.sh
|
|
TEST_PROGS += unicast_extensions.sh
|
|
TEST_PROGS += udpgro_fwd.sh
|
|
TEST_PROGS += udpgro_frglist.sh
|
|
TEST_PROGS += veth.sh
|
|
TEST_PROGS += ioam6.sh
|
|
TEST_PROGS += gro.sh
|
|
TEST_PROGS += gre_gso.sh
|
|
TEST_PROGS += cmsg_so_mark.sh
|
|
TEST_PROGS += cmsg_time.sh cmsg_ipv6.sh
|
|
TEST_PROGS += netns-name.sh
|
|
TEST_PROGS += srv6_end_dt46_l3vpn_test.sh
|
|
TEST_PROGS += srv6_end_dt4_l3vpn_test.sh
|
|
TEST_PROGS += srv6_end_dt6_l3vpn_test.sh
|
|
TEST_PROGS += srv6_hencap_red_l3vpn_test.sh
|
|
TEST_PROGS += srv6_hl2encap_red_l2vpn_test.sh
|
|
TEST_PROGS += srv6_end_next_csid_l3vpn_test.sh
|
|
TEST_PROGS += srv6_end_x_next_csid_l3vpn_test.sh
|
|
TEST_PROGS += srv6_end_flavors_test.sh
|
|
TEST_PROGS += vrf_strict_mode_test.sh
|
|
TEST_PROGS += arp_ndisc_evict_nocarrier.sh
|
|
TEST_PROGS += ndisc_unsolicited_na_test.sh
|
|
TEST_PROGS += arp_ndisc_untracked_subnets.sh
|
|
TEST_PROGS += stress_reuseport_listen.sh
|
|
TEST_PROGS += l2_tos_ttl_inherit.sh
|
|
TEST_PROGS += bind_bhash.sh
|
|
TEST_PROGS += ip_local_port_range.sh
|
|
TEST_PROGS += rps_default_mask.sh
|
|
TEST_PROGS += big_tcp.sh
|
|
TEST_PROGS_EXTENDED := toeplitz_client.sh toeplitz.sh
|
|
TEST_GEN_FILES = socket nettest
|
|
TEST_GEN_FILES += psock_fanout psock_tpacket msg_zerocopy reuseport_addr_any
|
|
TEST_GEN_FILES += tcp_mmap tcp_inq psock_snd txring_overwrite
|
|
TEST_GEN_FILES += udpgso udpgso_bench_tx udpgso_bench_rx ip_defrag
|
|
TEST_GEN_FILES += so_txtime ipv6_flowlabel ipv6_flowlabel_mgr so_netns_cookie
|
|
TEST_GEN_FILES += tcp_fastopen_backup_key
|
|
TEST_GEN_FILES += fin_ack_lat
|
|
TEST_GEN_FILES += reuseaddr_ports_exhausted
|
|
TEST_GEN_FILES += hwtstamp_config rxtimestamp timestamping txtimestamp
|
|
TEST_GEN_FILES += ipsec
|
|
TEST_GEN_FILES += ioam6_parser
|
|
TEST_GEN_FILES += gro
|
|
TEST_GEN_PROGS = reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa
|
|
TEST_GEN_PROGS += reuseport_dualstack reuseaddr_conflict tls tun tap
|
|
TEST_GEN_FILES += toeplitz
|
|
TEST_GEN_FILES += cmsg_sender
|
|
TEST_GEN_FILES += stress_reuseport_listen
|
|
TEST_PROGS += test_vxlan_vnifiltering.sh
|
|
TEST_GEN_FILES += io_uring_zerocopy_tx
|
|
TEST_PROGS += io_uring_zerocopy_tx.sh
|
|
TEST_GEN_FILES += bind_bhash
|
|
TEST_GEN_PROGS += sk_bind_sendto_listen
|
|
TEST_GEN_PROGS += sk_connect_zero_addr
|
|
TEST_PROGS += test_ingress_egress_chaining.sh
|
|
TEST_GEN_PROGS += so_incoming_cpu
|
|
TEST_PROGS += sctp_vrf.sh
|
|
TEST_GEN_FILES += sctp_hello
|
|
TEST_GEN_FILES += csum
|
|
TEST_GEN_FILES += nat6to4.o
|
|
TEST_GEN_FILES += xdp_dummy.o
|
|
TEST_GEN_FILES += ip_local_port_range
|
|
TEST_GEN_PROGS += bind_wildcard
|
|
TEST_GEN_PROGS += bind_timewait
|
|
TEST_PROGS += test_vxlan_mdb.sh
|
|
TEST_PROGS += test_bridge_neigh_suppress.sh
|
|
TEST_PROGS += test_vxlan_nolocalbypass.sh
|
|
TEST_PROGS += test_bridge_backup_port.sh
|
|
TEST_PROGS += ipv6_route_update_soft_lockup.sh
|
|
|
|
TEST_FILES := settings
|
|
TEST_FILES += in_netns.sh lib.sh net_helper.sh setup_loopback.sh setup_veth.sh
|
|
|
|
include ../lib.mk
|
|
|
|
$(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
|
|
$(OUTPUT)/tcp_mmap: LDLIBS += -lpthread -lcrypto
|
|
$(OUTPUT)/tcp_inq: LDLIBS += -lpthread
|
|
$(OUTPUT)/bind_bhash: LDLIBS += -lpthread
|
|
|
|
# Rules to generate bpf objs
|
|
CLANG ?= clang
|
|
SCRATCH_DIR := $(OUTPUT)/tools
|
|
BUILD_DIR := $(SCRATCH_DIR)/build
|
|
BPFDIR := $(abspath ../../../lib/bpf)
|
|
APIDIR := $(abspath ../../../include/uapi)
|
|
|
|
CCINCLUDE += -I../bpf
|
|
CCINCLUDE += -I../../../../usr/include/
|
|
CCINCLUDE += -I$(SCRATCH_DIR)/include
|
|
|
|
BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a
|
|
|
|
MAKE_DIRS := $(BUILD_DIR)/libbpf
|
|
$(MAKE_DIRS):
|
|
mkdir -p $@
|
|
|
|
# Get Clang's default includes on this system, as opposed to those seen by
|
|
# '--target=bpf'. This fixes "missing" files on some architectures/distros,
|
|
# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
|
|
#
|
|
# Use '-idirafter': Don't interfere with include mechanics except where the
|
|
# build would have failed anyways.
|
|
define get_sys_includes
|
|
$(shell $(1) $(2) -v -E - </dev/null 2>&1 \
|
|
| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
|
|
$(shell $(1) $(2) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
|
|
endef
|
|
|
|
ifneq ($(CROSS_COMPILE),)
|
|
CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
|
|
endif
|
|
|
|
CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
|
|
|
|
$(OUTPUT)/nat6to4.o $(OUTPUT)/xdp_dummy.o: $(OUTPUT)/%.o : %.c $(BPFOBJ) | $(MAKE_DIRS)
|
|
$(CLANG) -O2 --target=bpf -c $< $(CCINCLUDE) $(CLANG_SYS_INCLUDES) -o $@
|
|
|
|
$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
|
|
$(APIDIR)/linux/bpf.h \
|
|
| $(BUILD_DIR)/libbpf
|
|
$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \
|
|
EXTRA_CFLAGS='-g -O0' \
|
|
DESTDIR=$(SCRATCH_DIR) prefix= all install_headers
|
|
|
|
EXTRA_CLEAN := $(SCRATCH_DIR)
|