shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-09 12:17:12 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge e52a8b96c5 ("Merge branch 'selftests/bpf: Add parallelism to test_progs'") into android-mainline

Browse Source 
Steps on the way to 5.16-rc1

Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: I7fefeb83c0110a664e05aa8c1995406355673843
This commit is contained in:
Greg Kroah-Hartman
2021-11-09 17:01:36 +01:00
parent 59eb28e3f1 e52a8b96c5
commit 68db14aace
165 changed files with 6843 additions and 4169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Documentation/bpf/libbpf/libbpf_naming_convention.rst
View File

@@ -150,6 +150,46 @@ mirror of the mainline's version of libbpf for a stand-alone build.
However, all changes to libbpf's code base must be upstreamed through
the mainline kernel tree.
API documentation convention
============================
The libbpf API is documented via comments above definitions in
header files. These comments can be rendered by doxygen and sphinx
for well organized html output. This section describes the
convention in which these comments should be formated.
Here is an example from btf.h:
.. code-block:: c
/**
* @brief **btf__new()** creates a new instance of a BTF object from the raw
* bytes of an ELF's BTF section
* @param data raw bytes
* @param size number of bytes passed in `data`
* @return new BTF object instance which has to be eventually freed with
* **btf__free()**
*
* On error, error-code-encoded-as-pointer is returned, not a NULL. To extract
* error code from such a pointer `libbpf_get_error()` should be used. If
* `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is
* returned on error instead. In both cases thread-local `errno` variable is
* always set to error code as well.
*/
The comment must start with a block comment of the form '/\*\*'.
The documentation always starts with a @brief directive. This line is a short
description about this API. It starts with the name of the API, denoted in bold
like so: **api_name**. Please include an open and close parenthesis if this is a
function. Follow with the short description of the API. A longer form description
can be added below the last directive, at the bottom of the comment.
Parameters are denoted with the @param directive, there should be one for each
parameter. If this is a function with a non-void return, use the @return directive
to document it.
License
-------------------

1
MAINTAINERS
View File

@@ -3452,6 +3452,7 @@ S: Supported
F: arch/arm64/net/
BPF JIT for MIPS (32-BIT AND 64-BIT)
M: Johan Almbladh <johan.almbladh@anyfinetworks.com>
M: Paul Burton <paulburton@kernel.org>
L: netdev@vger.kernel.org
L: bpf@vger.kernel.org

5
arch/arm/net/bpf_jit_32.c
View File

@@ -1882,11 +1882,6 @@ static int validate_code(struct jit_ctx *ctx)
return 0;
}
void bpf_jit_compile(struct bpf_prog *prog)
{
/* Nothing to do here. We support Internal BPF. */
}
bool bpf_jit_needs_zext(void)
{
return true;

6
arch/mips/Kconfig
View File

@@ -56,7 +56,6 @@ config MIPS
select HAVE_ARCH_TRACEHOOK
select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES
select HAVE_ASM_MODVERSIONS
select HAVE_CBPF_JIT if !64BIT && !CPU_MICROMIPS
select HAVE_CONTEXT_TRACKING
select HAVE_TIF_NOHZ
select HAVE_C_RECORDMCOUNT
@@ -64,7 +63,10 @@ config MIPS
select HAVE_DEBUG_STACKOVERFLOW
select HAVE_DMA_CONTIGUOUS
select HAVE_DYNAMIC_FTRACE
select HAVE_EBPF_JIT if 64BIT && !CPU_MICROMIPS && TARGET_ISA_REV >= 2
select HAVE_EBPF_JIT if !CPU_MICROMIPS && \
!CPU_DADDI_WORKAROUNDS && \
!CPU_R4000_WORKAROUNDS && \
!CPU_R4400_WORKAROUNDS
select HAVE_EXIT_THREAD
select HAVE_FAST_GUP
select HAVE_FTRACE_MCOUNT_RECORD

5
arch/mips/include/asm/uasm.h
View File

@@ -145,6 +145,7 @@ Ip_u1(_mtlo);
Ip_u3u1u2(_mul);
Ip_u1u2(_multu);
Ip_u3u1u2(_mulu);
Ip_u3u1u2(_muhu);
Ip_u3u1u2(_nor);
Ip_u3u1u2(_or);
Ip_u2u1u3(_ori);
@@ -248,7 +249,11 @@ static inline void uasm_l##lb(struct uasm_label **lab, u32 *addr) \
#define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off)
#define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3)
#define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b)
#ifdef CONFIG_CPU_NOP_WORKAROUNDS
#define uasm_i_nop(buf) uasm_i_or(buf, 1, 1, 0)
#else
#define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0)
#endif
#define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1)
static inline void uasm_i_drotr_safe(u32 **p, unsigned int a1,

4
arch/mips/mm/uasm-mips.c
View File

@@ -90,7 +90,7 @@ static const struct insn insn_table[insn_invalid] = {
RS | RT | RD},
[insn_dmtc0] = {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
[insn_dmultu] = {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT},
[insn_dmulu] = {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op),
[insn_dmulu] = {M(spec_op, 0, 0, 0, dmultu_dmulu_op, dmultu_op),
RS | RT | RD},
[insn_drotr] = {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE},
[insn_drotr32] = {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE},
@@ -150,6 +150,8 @@ static const struct insn insn_table[insn_invalid] = {
[insn_mtlo] = {M(spec_op, 0, 0, 0, 0, mtlo_op), RS},
[insn_mulu] = {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op),
RS | RT | RD},
[insn_muhu] = {M(spec_op, 0, 0, 0, multu_muhu_op, multu_op),
RS | RT | RD},
#ifndef CONFIG_CPU_MIPSR6
[insn_mul] = {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD},
#else

3
arch/mips/mm/uasm.c
View File

@@ -59,7 +59,7 @@ enum opcode {
insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld,
insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi,
insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0,
insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor,
insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_muhu, insn_nor,
insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc,
insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra,
@@ -344,6 +344,7 @@ I_u1(_mtlo)
I_u3u1u2(_mul)
I_u1u2(_multu)
I_u3u1u2(_mulu)
I_u3u1u2(_muhu)
I_u3u1u2(_nor)
I_u3u1u2(_or)
I_u2u1u3(_ori)

9
arch/mips/net/Makefile
View File

@@ -1,5 +1,10 @@
# SPDX-License-Identifier: GPL-2.0-only
# MIPS networking code
obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o
obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o
obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp.o
ifeq ($(CONFIG_32BIT),y)
obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp32.o
else
obj-$(CONFIG_MIPS_EBPF_JIT) += bpf_jit_comp64.o
endif

1299
arch/mips/net/bpf_jit.c
View File

File diff suppressed because it is too large Load Diff

81
arch/mips/net/bpf_jit.h
View File

@@ -1,81 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Just-In-Time compiler for BPF filters on MIPS
*
* Copyright (c) 2014 Imagination Technologies Ltd.
* Author: Markos Chandras <markos.chandras@imgtec.com>
*/
#ifndef BPF_JIT_MIPS_OP_H
#define BPF_JIT_MIPS_OP_H
/* Registers used by JIT */
#define MIPS_R_ZERO 0
#define MIPS_R_V0 2
#define MIPS_R_A0 4
#define MIPS_R_A1 5
#define MIPS_R_T4 12
#define MIPS_R_T5 13
#define MIPS_R_T6 14
#define MIPS_R_T7 15
#define MIPS_R_S0 16
#define MIPS_R_S1 17
#define MIPS_R_S2 18
#define MIPS_R_S3 19
#define MIPS_R_S4 20
#define MIPS_R_S5 21
#define MIPS_R_S6 22
#define MIPS_R_S7 23
#define MIPS_R_SP 29
#define MIPS_R_RA 31
/* Conditional codes */
#define MIPS_COND_EQ 0x1
#define MIPS_COND_GE (0x1 << 1)
#define MIPS_COND_GT (0x1 << 2)
#define MIPS_COND_NE (0x1 << 3)
#define MIPS_COND_ALL (0x1 << 4)
/* Conditionals on X register or K immediate */
#define MIPS_COND_X (0x1 << 5)
#define MIPS_COND_K (0x1 << 6)
#define r_ret MIPS_R_V0
/*
* Use 2 scratch registers to avoid pipeline interlocks.
* There is no overhead during epilogue and prologue since
* any of the $s0-$s6 registers will only be preserved if
* they are going to actually be used.
*/
#define r_skb_hl MIPS_R_S0 /* skb header length */
#define r_skb_data MIPS_R_S1 /* skb actual data */
#define r_off MIPS_R_S2
#define r_A MIPS_R_S3
#define r_X MIPS_R_S4
#define r_skb MIPS_R_S5
#define r_M MIPS_R_S6
#define r_skb_len MIPS_R_S7
#define r_s0 MIPS_R_T4 /* scratch reg 1 */
#define r_s1 MIPS_R_T5 /* scratch reg 2 */
#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */
#define r_tmp MIPS_R_T7 /* No need to preserve this */
#define r_zero MIPS_R_ZERO
#define r_sp MIPS_R_SP
#define r_ra MIPS_R_RA
#ifndef __ASSEMBLY__
/* Declare ASM helpers */
#define DECLARE_LOAD_FUNC(func) \
extern u8 func(unsigned long *skb, int offset); \
extern u8 func##_negative(unsigned long *skb, int offset); \
extern u8 func##_positive(unsigned long *skb, int offset)
DECLARE_LOAD_FUNC(sk_load_word);
DECLARE_LOAD_FUNC(sk_load_half);
DECLARE_LOAD_FUNC(sk_load_byte);
#endif
#endif /* BPF_JIT_MIPS_OP_H */

285
arch/mips/net/bpf_jit_asm.S
View File

@@ -1,285 +0,0 @@
/*
* bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF
* compiler.
*
* Copyright (C) 2015 Imagination Technologies Ltd.
* Author: Markos Chandras <markos.chandras@imgtec.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation; version 2 of the License.
*/
#include <asm/asm.h>
#include <asm/isa-rev.h>
#include <asm/regdef.h>
#include "bpf_jit.h"
/* ABI
*
* r_skb_hl skb header length
* r_skb_data skb data
* r_off(a1) offset register
* r_A BPF register A
* r_X PF register X
* r_skb(a0) *skb
* r_M *scratch memory
* r_skb_le skb length
* r_s0 Scratch register 0
* r_s1 Scratch register 1
*
* On entry:
* a0: *skb
* a1: offset (imm or imm + X)
*
* All non-BPF-ABI registers are free for use. On return, we only
* care about r_ret. The BPF-ABI registers are assumed to remain
* unmodified during the entire filter operation.
*/
#define skb a0
#define offset a1
#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */
/* We know better :) so prevent assembler reordering etc */
.set noreorder
#define is_offset_negative(TYPE) \
/* If offset is negative we have more work to do */ \
slti t0, offset, 0; \
bgtz t0, bpf_slow_path_##TYPE##_neg; \
/* Be careful what follows in DS. */
#define is_offset_in_header(SIZE, TYPE) \
/* Reading from header? */ \
addiu $r_s0, $r_skb_hl, -SIZE; \
slt t0, $r_s0, offset; \
bgtz t0, bpf_slow_path_##TYPE; \
LEAF(sk_load_word)
is_offset_negative(word)
FEXPORT(sk_load_word_positive)
is_offset_in_header(4, word)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
.set reorder
lw $r_A, 0(t1)
.set noreorder
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
wsbh t0, $r_A
rotr $r_A, t0, 16
# else
sll t0, $r_A, 24
srl t1, $r_A, 24
srl t2, $r_A, 8
or t0, t0, t1
andi t2, t2, 0xff00
andi t1, $r_A, 0xff00
or t0, t0, t2
sll t1, t1, 8
or $r_A, t0, t1
# endif
#endif
jr $r_ra
move $r_ret, zero
END(sk_load_word)
LEAF(sk_load_half)
is_offset_negative(half)
FEXPORT(sk_load_half_positive)
is_offset_in_header(2, half)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
lhu $r_A, 0(t1)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
wsbh $r_A, $r_A
# else
sll t0, $r_A, 8
srl t1, $r_A, 8
andi t0, t0, 0xff00
or $r_A, t0, t1
# endif
#endif
jr $r_ra
move $r_ret, zero
END(sk_load_half)
LEAF(sk_load_byte)
is_offset_negative(byte)
FEXPORT(sk_load_byte_positive)
is_offset_in_header(1, byte)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
lbu $r_A, 0(t1)
jr $r_ra
move $r_ret, zero
END(sk_load_byte)
/*
* call skb_copy_bits:
* (prototype in linux/skbuff.h)
*
* int skb_copy_bits(sk_buff *skb, int offset, void *to, int len)
*
* o32 mandates we leave 4 spaces for argument registers in case
* the callee needs to use them. Even though we don't care about
* the argument registers ourselves, we need to allocate that space
* to remain ABI compliant since the callee may want to use that space.
* We also allocate 2 more spaces for $r_ra and our return register (*to).
*
* n64 is a bit different. The *caller* will allocate the space to preserve
* the arguments. So in 64-bit kernels, we allocate the 4-arg space for no
* good reason but it does not matter that much really.
*
* (void *to) is returned in r_s0
*
*/
#ifdef CONFIG_CPU_LITTLE_ENDIAN
#define DS_OFFSET(SIZE) (4 * SZREG)
#else
#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE))
#endif
#define bpf_slow_path_common(SIZE) \
/* Quick check. Are we within reasonable boundaries? */ \
LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \
sltu $r_s0, offset, $r_s1; \
beqz $r_s0, fault; \
/* Load 4th argument in DS */ \
LONG_ADDIU a3, zero, SIZE; \
PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
PTR_LA t0, skb_copy_bits; \
PTR_S $r_ra, (5 * SZREG)($r_sp); \
/* Assign low slot to a2 */ \
PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \
jalr t0; \
/* Reset our destination slot (DS but it's ok) */ \
INT_S zero, (4 * SZREG)($r_sp); \
/* \
* skb_copy_bits returns 0 on success and -EFAULT \
* on error. Our data live in a2. Do not bother with \
* our data if an error has been returned. \
*/ \
/* Restore our frame */ \
PTR_L $r_ra, (5 * SZREG)($r_sp); \
INT_L $r_s0, (4 * SZREG)($r_sp); \
bltz v0, fault; \
PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
move $r_ret, zero; \
NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp)
bpf_slow_path_common(4)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
wsbh t0, $r_s0
jr $r_ra
rotr $r_A, t0, 16
# else
sll t0, $r_s0, 24
srl t1, $r_s0, 24
srl t2, $r_s0, 8
or t0, t0, t1
andi t2, t2, 0xff00
andi t1, $r_s0, 0xff00
or t0, t0, t2
sll t1, t1, 8
jr $r_ra
or $r_A, t0, t1
# endif
#else
jr $r_ra
move $r_A, $r_s0
#endif
END(bpf_slow_path_word)
NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp)
bpf_slow_path_common(2)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
jr $r_ra
wsbh $r_A, $r_s0
# else
sll t0, $r_s0, 8
andi t1, $r_s0, 0xff00
andi t0, t0, 0xff00
srl t1, t1, 8
jr $r_ra
or $r_A, t0, t1
# endif
#else
jr $r_ra
move $r_A, $r_s0
#endif
END(bpf_slow_path_half)
NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp)
bpf_slow_path_common(1)
jr $r_ra
move $r_A, $r_s0
END(bpf_slow_path_byte)
/*
* Negative entry points
*/
.macro bpf_is_end_of_data
li t0, SKF_LL_OFF
/* Reading link layer data? */
slt t1, offset, t0
bgtz t1, fault
/* Be careful what follows in DS. */
.endm
/*
* call skb_copy_bits:
* (prototype in linux/filter.h)
*
* void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
* int k, unsigned int size)
*
* see above (bpf_slow_path_common) for ABI restrictions
*/
#define bpf_negative_common(SIZE) \
PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
PTR_LA t0, bpf_internal_load_pointer_neg_helper; \
PTR_S $r_ra, (5 * SZREG)($r_sp); \
jalr t0; \
li a2, SIZE; \
PTR_L $r_ra, (5 * SZREG)($r_sp); \
/* Check return pointer */ \
beqz v0, fault; \
PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
/* Preserve our pointer */ \
move $r_s0, v0; \
/* Set return value */ \
move $r_ret, zero; \
bpf_slow_path_word_neg:
bpf_is_end_of_data
NESTED(sk_load_word_negative, (6 * SZREG), $r_sp)
bpf_negative_common(4)
jr $r_ra
lw $r_A, 0($r_s0)
END(sk_load_word_negative)
bpf_slow_path_half_neg:
bpf_is_end_of_data
NESTED(sk_load_half_negative, (6 * SZREG), $r_sp)
bpf_negative_common(2)
jr $r_ra
lhu $r_A, 0($r_s0)
END(sk_load_half_negative)
bpf_slow_path_byte_neg:
bpf_is_end_of_data
NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp)
bpf_negative_common(1)
jr $r_ra
lbu $r_A, 0($r_s0)
END(sk_load_byte_negative)
fault:
jr $r_ra
addiu $r_ret, zero, 1

1034
arch/mips/net/bpf_jit_comp.c Normal file
View File

File diff suppressed because it is too large Load Diff

235
arch/mips/net/bpf_jit_comp.h Normal file
View File

@@ -0,0 +1,235 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Just-In-Time compiler for eBPF bytecode on 32-bit and 64-bit MIPS.
*
* Copyright (c) 2021 Anyfi Networks AB.
* Author: Johan Almbladh <johan.almbladh@gmail.com>
*
* Based on code and ideas from
* Copyright (c) 2017 Cavium, Inc.
* Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com>
* Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com>
*/
#ifndef _BPF_JIT_COMP_H
#define _BPF_JIT_COMP_H
/* MIPS registers */
#define MIPS_R_ZERO 0 /* Const zero */
#define MIPS_R_AT 1 /* Asm temp */
#define MIPS_R_V0 2 /* Result */
#define MIPS_R_V1 3 /* Result */
#define MIPS_R_A0 4 /* Argument */
#define MIPS_R_A1 5 /* Argument */
#define MIPS_R_A2 6 /* Argument */
#define MIPS_R_A3 7 /* Argument */
#define MIPS_R_A4 8 /* Arg (n64) */
#define MIPS_R_A5 9 /* Arg (n64) */
#define MIPS_R_A6 10 /* Arg (n64) */
#define MIPS_R_A7 11 /* Arg (n64) */
#define MIPS_R_T0 8 /* Temp (o32) */
#define MIPS_R_T1 9 /* Temp (o32) */
#define MIPS_R_T2 10 /* Temp (o32) */
#define MIPS_R_T3 11 /* Temp (o32) */
#define MIPS_R_T4 12 /* Temporary */
#define MIPS_R_T5 13 /* Temporary */
#define MIPS_R_T6 14 /* Temporary */
#define MIPS_R_T7 15 /* Temporary */
#define MIPS_R_S0 16 /* Saved */
#define MIPS_R_S1 17 /* Saved */
#define MIPS_R_S2 18 /* Saved */
#define MIPS_R_S3 19 /* Saved */
#define MIPS_R_S4 20 /* Saved */
#define MIPS_R_S5 21 /* Saved */
#define MIPS_R_S6 22 /* Saved */
#define MIPS_R_S7 23 /* Saved */
#define MIPS_R_T8 24 /* Temporary */
#define MIPS_R_T9 25 /* Temporary */
/* MIPS_R_K0 26 Reserved */
/* MIPS_R_K1 27 Reserved */
#define MIPS_R_GP 28 /* Global ptr */
#define MIPS_R_SP 29 /* Stack ptr */
#define MIPS_R_FP 30 /* Frame ptr */
#define MIPS_R_RA 31 /* Return */
/*
* Jump address mask for immediate jumps. The four most significant bits
* must be equal to PC.
*/
#define MIPS_JMP_MASK 0x0fffffffUL
/* Maximum number of iterations in offset table computation */
#define JIT_MAX_ITERATIONS 8
/*
* Jump pseudo-instructions used internally
* for branch conversion and branch optimization.
*/
#define JIT_JNSET 0xe0
#define JIT_JNOP 0xf0
/* Descriptor flag for PC-relative branch conversion */
#define JIT_DESC_CONVERT BIT(31)
/* JIT context for an eBPF program */
struct jit_context {
struct bpf_prog *program; /* The eBPF program being JITed */
u32 *descriptors; /* eBPF to JITed CPU insn descriptors */
u32 *target; /* JITed code buffer */
u32 bpf_index; /* Index of current BPF program insn */
u32 jit_index; /* Index of current JIT target insn */
u32 changes; /* Number of PC-relative branch conv */
u32 accessed; /* Bit mask of read eBPF registers */
u32 clobbered; /* Bit mask of modified CPU registers */
u32 stack_size; /* Total allocated stack size in bytes */
u32 saved_size; /* Size of callee-saved registers */
u32 stack_used; /* Stack size used for function calls */
};
/* Emit the instruction if the JIT memory space has been allocated */
#define __emit(ctx, func, ...) \
do { \
if ((ctx)->target != NULL) { \
u32 *p = &(ctx)->target[ctx->jit_index]; \
uasm_i_##func(&p, ##__VA_ARGS__); \
} \
(ctx)->jit_index++; \
} while (0)
#define emit(...) __emit(__VA_ARGS__)
/* Workaround for R10000 ll/sc errata */
#ifdef CONFIG_WAR_R10000
#define LLSC_beqz beqzl
#else
#define LLSC_beqz beqz
#endif
/* Workaround for Loongson-3 ll/sc errata */
#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS
#define LLSC_sync(ctx) emit(ctx, sync, 0)
#define LLSC_offset 4
#else
#define LLSC_sync(ctx)
#define LLSC_offset 0
#endif
/* Workaround for Loongson-2F jump errata */
#ifdef CONFIG_CPU_JUMP_WORKAROUNDS
#define JALR_MASK 0xffffffffcfffffffULL
#else
#define JALR_MASK (~0ULL)
#endif
/*
* Mark a BPF register as accessed, it needs to be
* initialized by the program if expected, e.g. FP.
*/
static inline void access_reg(struct jit_context *ctx, u8 reg)
{
ctx->accessed |= BIT(reg);
}
/*
* Mark a CPU register as clobbered, it needs to be
* saved/restored by the program if callee-saved.
*/
static inline void clobber_reg(struct jit_context *ctx, u8 reg)
{
ctx->clobbered |= BIT(reg);
}
/*
* Push registers on the stack, starting at a given depth from the stack
* pointer and increasing. The next depth to be written is returned.
*/
int push_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth);
/*
* Pop registers from the stack, starting at a given depth from the stack
* pointer and increasing. The next depth to be read is returned.
*/
int pop_regs(struct jit_context *ctx, u32 mask, u32 excl, int depth);
/* Compute the 28-bit jump target address from a BPF program location */
int get_target(struct jit_context *ctx, u32 loc);
/* Compute the PC-relative offset to relative BPF program offset */
int get_offset(const struct jit_context *ctx, int off);
/* dst = imm (32-bit) */
void emit_mov_i(struct jit_context *ctx, u8 dst, s32 imm);
/* dst = src (32-bit) */
void emit_mov_r(struct jit_context *ctx, u8 dst, u8 src);
/* Validate ALU/ALU64 immediate range */
bool valid_alu_i(u8 op, s32 imm);
/* Rewrite ALU/ALU64 immediate operation */
bool rewrite_alu_i(u8 op, s32 imm, u8 *alu, s32 *val);
/* ALU immediate operation (32-bit) */
void emit_alu_i(struct jit_context *ctx, u8 dst, s32 imm, u8 op);
/* ALU register operation (32-bit) */
void emit_alu_r(struct jit_context *ctx, u8 dst, u8 src, u8 op);
/* Atomic read-modify-write (32-bit) */
void emit_atomic_r(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 code);
/* Atomic compare-and-exchange (32-bit) */
void emit_cmpxchg_r(struct jit_context *ctx, u8 dst, u8 src, u8 res, s16 off);
/* Swap bytes and truncate a register word or half word */
void emit_bswap_r(struct jit_context *ctx, u8 dst, u32 width);
/* Validate JMP/JMP32 immediate range */
bool valid_jmp_i(u8 op, s32 imm);
/* Prepare a PC-relative jump operation with immediate conditional */
void setup_jmp_i(struct jit_context *ctx, s32 imm, u8 width,
u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off);
/* Prepare a PC-relative jump operation with register conditional */
void setup_jmp_r(struct jit_context *ctx, bool same_reg,
u8 bpf_op, s16 bpf_off, u8 *jit_op, s32 *jit_off);
/* Finish a PC-relative jump operation */
int finish_jmp(struct jit_context *ctx, u8 jit_op, s16 bpf_off);
/* Conditional JMP/JMP32 immediate */
void emit_jmp_i(struct jit_context *ctx, u8 dst, s32 imm, s32 off, u8 op);
/* Conditional JMP/JMP32 register */
void emit_jmp_r(struct jit_context *ctx, u8 dst, u8 src, s32 off, u8 op);
/* Jump always */
int emit_ja(struct jit_context *ctx, s16 off);
/* Jump to epilogue */
int emit_exit(struct jit_context *ctx);
/*
* Build program prologue to set up the stack and registers.
* This function is implemented separately for 32-bit and 64-bit JITs.
*/
void build_prologue(struct jit_context *ctx);
/*
* Build the program epilogue to restore the stack and registers.
* This function is implemented separately for 32-bit and 64-bit JITs.
*/
void build_epilogue(struct jit_context *ctx, int dest_reg);
/*
* Convert an eBPF instruction to native instruction, i.e
* JITs an eBPF instruction.
* Returns :
* 0 - Successfully JITed an 8-byte eBPF instruction
* >0 - Successfully JITed a 16-byte eBPF instruction
* <0 - Failed to JIT.
* This function is implemented separately for 32-bit and 64-bit JITs.
*/
int build_insn(const struct bpf_insn *insn, struct jit_context *ctx);
#endif /* _BPF_JIT_COMP_H */

1899
arch/mips/net/bpf_jit_comp32.c Normal file
View File

File diff suppressed because it is too large Load Diff

1060
arch/mips/net/bpf_jit_comp64.c Normal file
View File

File diff suppressed because it is too large Load Diff

1938
arch/mips/net/ebpf_jit.c
View File

File diff suppressed because it is too large Load Diff

124
arch/x86/net/bpf_jit_comp.c
View File

@@ -697,6 +697,20 @@ static void maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64)
*pprog = prog;
}
/*
* Similar version of maybe_emit_mod() for a single register
*/
static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64)
{
u8 *prog = *pprog;
if (is64)
EMIT1(add_1mod(0x48, reg));
else if (is_ereg(reg))
EMIT1(add_1mod(0x40, reg));
*pprog = prog;
}
/* LDX: dst_reg = *(u8*)(src_reg + off) */
static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
@@ -925,10 +939,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
/* neg dst */
case BPF_ALU | BPF_NEG:
case BPF_ALU64 | BPF_NEG:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
maybe_emit_1mod(&prog, dst_reg,
BPF_CLASS(insn->code) == BPF_ALU64);
EMIT2(0xF7, add_1reg(0xD8, dst_reg));
break;
@@ -942,10 +954,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
case BPF_ALU64 | BPF_AND | BPF_K:
case BPF_ALU64 | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
maybe_emit_1mod(&prog, dst_reg,
BPF_CLASS(insn->code) == BPF_ALU64);
/*
* b3 holds 'normal' opcode, b2 short form only valid
@@ -1002,19 +1012,30 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
case BPF_ALU64 | BPF_MOD | BPF_X:
case BPF_ALU64 | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
EMIT1(0x50); /* push rax */
EMIT1(0x52); /* push rdx */
case BPF_ALU64 | BPF_DIV | BPF_K: {
bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
if (BPF_SRC(insn->code) == BPF_X)
/* mov r11, src_reg */
EMIT_mov(AUX_REG, src_reg);
else
if (dst_reg != BPF_REG_0)
EMIT1(0x50); /* push rax */
if (dst_reg != BPF_REG_3)
EMIT1(0x52); /* push rdx */
if (BPF_SRC(insn->code) == BPF_X) {
if (src_reg == BPF_REG_0 ||
src_reg == BPF_REG_3) {
/* mov r11, src_reg */
EMIT_mov(AUX_REG, src_reg);
src_reg = AUX_REG;
}
} else {
/* mov r11, imm32 */
EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
src_reg = AUX_REG;
}
/* mov rax, dst_reg */
EMIT_mov(BPF_REG_0, dst_reg);
if (dst_reg != BPF_REG_0)
/* mov rax, dst_reg */
emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg);
/*
* xor edx, edx
@@ -1022,33 +1043,30 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
*/
EMIT2(0x31, 0xd2);
if (BPF_CLASS(insn->code) == BPF_ALU64)
/* div r11 */
EMIT3(0x49, 0xF7, 0xF3);
else
/* div r11d */
EMIT3(0x41, 0xF7, 0xF3);
/* div src_reg */
maybe_emit_1mod(&prog, src_reg, is64);
EMIT2(0xF7, add_1reg(0xF0, src_reg));
if (BPF_OP(insn->code) == BPF_MOD)
/* mov r11, rdx */
EMIT3(0x49, 0x89, 0xD3);
else
/* mov r11, rax */
EMIT3(0x49, 0x89, 0xC3);
if (BPF_OP(insn->code) == BPF_MOD &&
dst_reg != BPF_REG_3)
/* mov dst_reg, rdx */
emit_mov_reg(&prog, is64, dst_reg, BPF_REG_3);
else if (BPF_OP(insn->code) == BPF_DIV &&
dst_reg != BPF_REG_0)
/* mov dst_reg, rax */
emit_mov_reg(&prog, is64, dst_reg, BPF_REG_0);
EMIT1(0x5A); /* pop rdx */
EMIT1(0x58); /* pop rax */
/* mov dst_reg, r11 */
EMIT_mov(dst_reg, AUX_REG);
if (dst_reg != BPF_REG_3)
EMIT1(0x5A); /* pop rdx */
if (dst_reg != BPF_REG_0)
EMIT1(0x58); /* pop rax */
break;
}
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU64 | BPF_MUL | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_2mod(0x48, dst_reg, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_2mod(0x40, dst_reg, dst_reg));
maybe_emit_mod(&prog, dst_reg, dst_reg,
BPF_CLASS(insn->code) == BPF_ALU64);
if (is_imm8(imm32))
/* imul dst_reg, dst_reg, imm8 */
@@ -1063,10 +1081,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
case BPF_ALU | BPF_MUL | BPF_X:
case BPF_ALU64 | BPF_MUL | BPF_X:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_2mod(0x48, src_reg, dst_reg));
else if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT1(add_2mod(0x40, src_reg, dst_reg));
maybe_emit_mod(&prog, src_reg, dst_reg,
BPF_CLASS(insn->code) == BPF_ALU64);
/* imul dst_reg, src_reg */
EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));
@@ -1079,10 +1095,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
case BPF_ALU64 | BPF_LSH | BPF_K:
case BPF_ALU64 | BPF_RSH | BPF_K:
case BPF_ALU64 | BPF_ARSH | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
maybe_emit_1mod(&prog, dst_reg,
BPF_CLASS(insn->code) == BPF_ALU64);
b3 = simple_alu_opcodes[BPF_OP(insn->code)];
if (imm32 == 1)
@@ -1113,10 +1127,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
}
/* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
maybe_emit_1mod(&prog, dst_reg,
BPF_CLASS(insn->code) == BPF_ALU64);
b3 = simple_alu_opcodes[BPF_OP(insn->code)];
EMIT2(0xD3, add_1reg(b3, dst_reg));
@@ -1423,10 +1435,8 @@ st: if (is_imm8(insn->off))
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP32 | BPF_JSET | BPF_K:
/* test dst_reg, imm32 */
if (BPF_CLASS(insn->code) == BPF_JMP)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
maybe_emit_1mod(&prog, dst_reg,
BPF_CLASS(insn->code) == BPF_JMP);
EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
goto emit_cond_jmp;
@@ -1459,10 +1469,8 @@ st: if (is_imm8(insn->off))
}
/* cmp dst_reg, imm8/32 */
if (BPF_CLASS(insn->code) == BPF_JMP)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
maybe_emit_1mod(&prog, dst_reg,
BPF_CLASS(insn->code) == BPF_JMP);
if (is_imm8(imm32))
EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);

8
include/linux/bpf.h
View File

@@ -513,7 +513,7 @@ struct bpf_verifier_ops {
const struct btf_type *t, int off, int size,
enum bpf_access_type atype,
u32 *next_btf_id);
bool (*check_kfunc_call)(u32 kfunc_btf_id);
bool (*check_kfunc_call)(u32 kfunc_btf_id, struct module *owner);
};
struct bpf_prog_offload_ops {
@@ -877,6 +877,7 @@ struct bpf_prog_aux {
void *jit_data; /* JIT specific data. arch dependent */
struct bpf_jit_poke_descriptor *poke_tab;
struct bpf_kfunc_desc_tab *kfunc_tab;
struct bpf_kfunc_btf_tab *kfunc_btf_tab;
u32 size_poke_tab;
struct bpf_ksym ksym;
const struct bpf_prog_ops *ops;
@@ -1642,7 +1643,7 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
const union bpf_attr *kattr,
union bpf_attr __user *uattr);
bool bpf_prog_test_check_kfunc_call(u32 kfunc_id);
bool bpf_prog_test_check_kfunc_call(u32 kfunc_id, struct module *owner);
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
const struct bpf_prog *prog,
struct bpf_insn_access_aux *info);
@@ -1863,7 +1864,8 @@ static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
return -ENOTSUPP;
}
static inline bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
static inline bool bpf_prog_test_check_kfunc_call(u32 kfunc_id,
struct module *owner)
{
return false;
}

2
include/linux/bpf_verifier.h
View File

@@ -527,5 +527,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *tgt_prog,
u32 btf_id,
struct bpf_attach_target_info *tgt_info);
void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
#endif /* _LINUX_BPF_VERIFIER_H */

1
include/linux/bpfptr.h
View File

@@ -3,6 +3,7 @@
#ifndef _LINUX_BPFPTR_H
#define _LINUX_BPFPTR_H
#include <linux/mm.h>
#include <linux/sockptr.h>
typedef sockptr_t bpfptr_t;

39
include/linux/btf.h
View File

@@ -5,6 +5,7 @@
#define _LINUX_BTF_H 1
#include <linux/types.h>
#include <linux/bpfptr.h>
#include <uapi/linux/btf.h>
#include <uapi/linux/bpf.h>
@@ -238,4 +239,42 @@ static inline const char *btf_name_by_offset(const struct btf *btf,
}
#endif
struct kfunc_btf_id_set {
struct list_head list;
struct btf_id_set *set;
struct module *owner;
};
struct kfunc_btf_id_list;
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
void register_kfunc_btf_id_set(struct kfunc_btf_id_list *l,
struct kfunc_btf_id_set *s);
void unregister_kfunc_btf_id_set(struct kfunc_btf_id_list *l,
struct kfunc_btf_id_set *s);
bool bpf_check_mod_kfunc_call(struct kfunc_btf_id_list *klist, u32 kfunc_id,
struct module *owner);
#else
static inline void register_kfunc_btf_id_set(struct kfunc_btf_id_list *l,
struct kfunc_btf_id_set *s)
{
}
static inline void unregister_kfunc_btf_id_set(struct kfunc_btf_id_list *l,
struct kfunc_btf_id_set *s)
{
}
static inline bool bpf_check_mod_kfunc_call(struct kfunc_btf_id_list *klist,
u32 kfunc_id, struct module *owner)
{
return false;
}
#endif
#define DEFINE_KFUNC_BTF_ID_SET(set, name) \
struct kfunc_btf_id_set name = { LIST_HEAD_INIT(name.list), (set), \
THIS_MODULE }
extern struct kfunc_btf_id_list bpf_tcp_ca_kfunc_list;
extern struct kfunc_btf_id_list prog_test_kfunc_list;
#endif

19
include/trace/bpf_probe.h
View File

@@ -93,8 +93,7 @@ __section("__bpf_raw_tp_map") = { \
#define FIRST(x, ...) x
#undef DEFINE_EVENT_WRITABLE
#define DEFINE_EVENT_WRITABLE(template, call, proto, args, size) \
#define __CHECK_WRITABLE_BUF_SIZE(call, proto, args, size) \
static inline void bpf_test_buffer_##call(void) \
{ \
/* BUILD_BUG_ON() is ignored if the code is completely eliminated, but \
@@ -103,8 +102,12 @@ static inline void bpf_test_buffer_##call(void) \
*/ \
FIRST(proto); \
(void)BUILD_BUG_ON_ZERO(size != sizeof(*FIRST(args))); \
} \
__DEFINE_EVENT(template, call, PARAMS(proto), PARAMS(args), size)
}
#undef DEFINE_EVENT_WRITABLE
#define DEFINE_EVENT_WRITABLE(template, call, proto, args, size) \
__CHECK_WRITABLE_BUF_SIZE(call, PARAMS(proto), PARAMS(args), size) \
__DEFINE_EVENT(template, call, PARAMS(proto), PARAMS(args), size)
#undef DEFINE_EVENT
#define DEFINE_EVENT(template, call, proto, args) \
@@ -119,9 +122,17 @@ __DEFINE_EVENT(template, call, PARAMS(proto), PARAMS(args), size)
__BPF_DECLARE_TRACE(call, PARAMS(proto), PARAMS(args)) \
__DEFINE_EVENT(call, call, PARAMS(proto), PARAMS(args), 0)
#undef DECLARE_TRACE_WRITABLE
#define DECLARE_TRACE_WRITABLE(call, proto, args, size) \
__CHECK_WRITABLE_BUF_SIZE(call, PARAMS(proto), PARAMS(args), size) \
__BPF_DECLARE_TRACE(call, PARAMS(proto), PARAMS(args)) \
__DEFINE_EVENT(call, call, PARAMS(proto), PARAMS(args), size)
#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
#undef DECLARE_TRACE_WRITABLE
#undef DEFINE_EVENT_WRITABLE
#undef __CHECK_WRITABLE_BUF_SIZE
#undef __DEFINE_EVENT
#undef FIRST

55
kernel/bpf/btf.c
View File

@@ -6343,3 +6343,58 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
};
BTF_ID_LIST_GLOBAL_SINGLE(btf_task_struct_ids, struct, task_struct)
/* BTF ID set registration API for modules */
struct kfunc_btf_id_list {
struct list_head list;
struct mutex mutex;
};
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
void register_kfunc_btf_id_set(struct kfunc_btf_id_list *l,
struct kfunc_btf_id_set *s)
{
mutex_lock(&l->mutex);
list_add(&s->list, &l->list);
mutex_unlock(&l->mutex);
}
EXPORT_SYMBOL_GPL(register_kfunc_btf_id_set);
void unregister_kfunc_btf_id_set(struct kfunc_btf_id_list *l,
struct kfunc_btf_id_set *s)
{
mutex_lock(&l->mutex);
list_del_init(&s->list);
mutex_unlock(&l->mutex);
}
EXPORT_SYMBOL_GPL(unregister_kfunc_btf_id_set);
bool bpf_check_mod_kfunc_call(struct kfunc_btf_id_list *klist, u32 kfunc_id,
struct module *owner)
{
struct kfunc_btf_id_set *s;
if (!owner)
return false;
mutex_lock(&klist->mutex);
list_for_each_entry(s, &klist->list, list) {
if (s->owner == owner && btf_id_set_contains(s->set, kfunc_id)) {
mutex_unlock(&klist->mutex);
return true;
}
}
mutex_unlock(&klist->mutex);
return false;
}
#endif
#define DEFINE_KFUNC_BTF_ID_LIST(name) \
struct kfunc_btf_id_list name = { LIST_HEAD_INIT(name.list), \
__MUTEX_INITIALIZER(name.mutex) }; \
EXPORT_SYMBOL_GPL(name)
DEFINE_KFUNC_BTF_ID_LIST(bpf_tcp_ca_kfunc_list);
DEFINE_KFUNC_BTF_ID_LIST(prog_test_kfunc_list);

4
kernel/bpf/core.c
View File

@@ -32,6 +32,7 @@
#include <linux/perf_event.h>
#include <linux/extable.h>
#include <linux/log2.h>
#include <linux/bpf_verifier.h>
#include <asm/barrier.h>
#include <asm/unaligned.h>
@@ -2263,6 +2264,9 @@ static void bpf_prog_free_deferred(struct work_struct *work)
int i;
aux = container_of(work, struct bpf_prog_aux, work);
#ifdef CONFIG_BPF_SYSCALL
bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab);
#endif
bpf_free_used_maps(aux);
bpf_free_used_btfs(aux);
if (bpf_prog_is_dev_bound(aux))

25
kernel/bpf/preload/Makefile
View File

@@ -1,21 +1,36 @@
# SPDX-License-Identifier: GPL-2.0
LIBBPF_SRCS = $(srctree)/tools/lib/bpf/
LIBBPF_A = $(obj)/libbpf.a
LIBBPF_OUT = $(abspath $(obj))
LIBBPF_OUT = $(abspath $(obj))/libbpf
LIBBPF_A = $(LIBBPF_OUT)/libbpf.a
LIBBPF_DESTDIR = $(LIBBPF_OUT)
LIBBPF_INCLUDE = $(LIBBPF_DESTDIR)/include
# Although not in use by libbpf's Makefile, set $(O) so that the "dummy" test
# in tools/scripts/Makefile.include always succeeds when building the kernel
# with $(O) pointing to a relative path, as in "make O=build bindeb-pkg".
$(LIBBPF_A):
$(Q)$(MAKE) -C $(LIBBPF_SRCS) O=$(LIBBPF_OUT)/ OUTPUT=$(LIBBPF_OUT)/ $(LIBBPF_OUT)/libbpf.a
$(LIBBPF_A): | $(LIBBPF_OUT)
$(Q)$(MAKE) -C $(LIBBPF_SRCS) O=$(LIBBPF_OUT)/ OUTPUT=$(LIBBPF_OUT)/ \
DESTDIR=$(LIBBPF_DESTDIR) prefix= \
$(LIBBPF_OUT)/libbpf.a install_headers
libbpf_hdrs: $(LIBBPF_A)
.PHONY: libbpf_hdrs
$(LIBBPF_OUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $@
userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \
-I $(srctree)/tools/lib/ -Wno-unused-result
-I $(LIBBPF_INCLUDE) -Wno-unused-result
userprogs := bpf_preload_umd
clean-files := $(userprogs) bpf_helper_defs.h FEATURE-DUMP.libbpf staticobjs/ feature/
clean-files += $(LIBBPF_OUT) $(LIBBPF_DESTDIR)
$(obj)/iterators/iterators.o: | libbpf_hdrs
bpf_preload_umd-objs := iterators/iterators.o
bpf_preload_umd-userldlibs := $(LIBBPF_A) -lelf -lz

38
kernel/bpf/preload/iterators/Makefile
View File

@@ -1,18 +1,26 @@
# SPDX-License-Identifier: GPL-2.0
OUTPUT := .output
abs_out := $(abspath $(OUTPUT))
CLANG ?= clang
LLC ?= llc
LLVM_STRIP ?= llvm-strip
TOOLS_PATH := $(abspath ../../../../tools)
BPFTOOL_SRC := $(TOOLS_PATH)/bpf/bpftool
BPFTOOL_OUTPUT := $(abs_out)/bpftool
DEFAULT_BPFTOOL := $(OUTPUT)/sbin/bpftool
BPFTOOL ?= $(DEFAULT_BPFTOOL)
LIBBPF_SRC := $(abspath ../../../../tools/lib/bpf)
BPFOBJ := $(OUTPUT)/libbpf.a
BPF_INCLUDE := $(OUTPUT)
INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../../../tools/lib) \
-I$(abspath ../../../../tools/include/uapi)
LIBBPF_SRC := $(TOOLS_PATH)/lib/bpf
LIBBPF_OUTPUT := $(abs_out)/libbpf
LIBBPF_DESTDIR := $(LIBBPF_OUTPUT)
LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)/include
BPFOBJ := $(LIBBPF_OUTPUT)/libbpf.a
INCLUDES := -I$(OUTPUT) -I$(LIBBPF_INCLUDE) -I$(TOOLS_PATH)/include/uapi
CFLAGS := -g -Wall
abs_out := $(abspath $(OUTPUT))
ifeq ($(V),1)
Q =
msg =
@@ -44,14 +52,18 @@ $(OUTPUT)/iterators.bpf.o: iterators.bpf.c $(BPFOBJ) | $(OUTPUT)
-c $(filter %.c,$^) -o $@ && \
$(LLVM_STRIP) -g $@
$(OUTPUT):
$(OUTPUT) $(LIBBPF_OUTPUT) $(BPFTOOL_OUTPUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $(OUTPUT)
$(Q)mkdir -p $@
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) \
OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
OUTPUT=$(abspath $(dir $@))/ prefix= \
DESTDIR=$(LIBBPF_DESTDIR) $(abspath $@) install_headers
$(DEFAULT_BPFTOOL):
$(Q)$(MAKE) $(submake_extras) -C ../../../../tools/bpf/bpftool \
prefix= OUTPUT=$(abs_out)/ DESTDIR=$(abs_out) install
$(DEFAULT_BPFTOOL): $(BPFOBJ) | $(BPFTOOL_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C $(BPFTOOL_SRC) \
OUTPUT=$(BPFTOOL_OUTPUT)/ \
LIBBPF_OUTPUT=$(LIBBPF_OUTPUT)/ \
LIBBPF_DESTDIR=$(LIBBPF_DESTDIR)/ \
prefix= DESTDIR=$(abs_out)/ install-bin

231
kernel/bpf/verifier.c
View File

@@ -1640,52 +1640,173 @@ static int add_subprog(struct bpf_verifier_env *env, int off)
return env->subprog_cnt - 1;
}
#define MAX_KFUNC_DESCS 256
#define MAX_KFUNC_BTFS 256
struct bpf_kfunc_desc {
struct btf_func_model func_model;
u32 func_id;
s32 imm;
u16 offset;
};
struct bpf_kfunc_btf {
struct btf *btf;
struct module *module;
u16 offset;
};
#define MAX_KFUNC_DESCS 256
struct bpf_kfunc_desc_tab {
struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
u32 nr_descs;
};
static int kfunc_desc_cmp_by_id(const void *a, const void *b)
struct bpf_kfunc_btf_tab {
struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
u32 nr_descs;
};
static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
{
const struct bpf_kfunc_desc *d0 = a;
const struct bpf_kfunc_desc *d1 = b;
/* func_id is not greater than BTF_MAX_TYPE */
return d0->func_id - d1->func_id;
return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
}
static int kfunc_btf_cmp_by_off(const void *a, const void *b)
{
const struct bpf_kfunc_btf *d0 = a;
const struct bpf_kfunc_btf *d1 = b;
return d0->offset - d1->offset;
}
static const struct bpf_kfunc_desc *
find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
{
struct bpf_kfunc_desc desc = {
.func_id = func_id,
.offset = offset,
};
struct bpf_kfunc_desc_tab *tab;
tab = prog->aux->kfunc_tab;
return bsearch(&desc, tab->descs, tab->nr_descs,
sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
}
static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
s16 offset, struct module **btf_modp)
{
struct bpf_kfunc_btf kf_btf = { .offset = offset };
struct bpf_kfunc_btf_tab *tab;
struct bpf_kfunc_btf *b;
struct module *mod;
struct btf *btf;
int btf_fd;
tab = env->prog->aux->kfunc_btf_tab;
b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
if (!b) {
if (tab->nr_descs == MAX_KFUNC_BTFS) {
verbose(env, "too many different module BTFs\n");
return ERR_PTR(-E2BIG);
}
if (bpfptr_is_null(env->fd_array)) {
verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
return ERR_PTR(-EPROTO);
}
if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
offset * sizeof(btf_fd),
sizeof(btf_fd)))
return ERR_PTR(-EFAULT);
btf = btf_get_by_fd(btf_fd);
if (IS_ERR(btf))
return btf;
if (!btf_is_module(btf)) {
verbose(env, "BTF fd for kfunc is not a module BTF\n");
btf_put(btf);
return ERR_PTR(-EINVAL);
}
mod = btf_try_get_module(btf);
if (!mod) {
btf_put(btf);
return ERR_PTR(-ENXIO);
}
b = &tab->descs[tab->nr_descs++];
b->btf = btf;
b->module = mod;
b->offset = offset;
sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
kfunc_btf_cmp_by_off, NULL);
}
if (btf_modp)
*btf_modp = b->module;
return b->btf;
}
void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
{
if (!tab)
return;
while (tab->nr_descs--) {
module_put(tab->descs[tab->nr_descs].module);
btf_put(tab->descs[tab->nr_descs].btf);
}
kfree(tab);
}
static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env,
u32 func_id, s16 offset,
struct module **btf_modp)
{
struct btf *kfunc_btf;
if (offset) {
if (offset < 0) {
/* In the future, this can be allowed to increase limit
* of fd index into fd_array, interpreted as u16.
*/
verbose(env, "negative offset disallowed for kernel module function call\n");
return ERR_PTR(-EINVAL);
}
kfunc_btf = __find_kfunc_desc_btf(env, offset, btf_modp);
if (IS_ERR_OR_NULL(kfunc_btf)) {
verbose(env, "cannot find module BTF for func_id %u\n", func_id);
return kfunc_btf ?: ERR_PTR(-ENOENT);
}
return kfunc_btf;
}
return btf_vmlinux ?: ERR_PTR(-ENOENT);
}
static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
{
const struct btf_type *func, *func_proto;
struct bpf_kfunc_btf_tab *btf_tab;
struct bpf_kfunc_desc_tab *tab;
struct bpf_prog_aux *prog_aux;
struct bpf_kfunc_desc *desc;
const char *func_name;
struct btf *desc_btf;
unsigned long addr;
int err;
prog_aux = env->prog->aux;
tab = prog_aux->kfunc_tab;
btf_tab = prog_aux->kfunc_btf_tab;
if (!tab) {
if (!btf_vmlinux) {
verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
@@ -1713,7 +1834,29 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
prog_aux->kfunc_tab = tab;
}
if (find_kfunc_desc(env->prog, func_id))
/* func_id == 0 is always invalid, but instead of returning an error, be
* conservative and wait until the code elimination pass before returning
* error, so that invalid calls that get pruned out can be in BPF programs
* loaded from userspace. It is also required that offset be untouched
* for such calls.
*/
if (!func_id && !offset)
return 0;
if (!btf_tab && offset) {
btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
if (!btf_tab)
return -ENOMEM;
prog_aux->kfunc_btf_tab = btf_tab;
}
desc_btf = find_kfunc_desc_btf(env, func_id, offset, NULL);
if (IS_ERR(desc_btf)) {
verbose(env, "failed to find BTF for kernel function\n");
return PTR_ERR(desc_btf);
}
if (find_kfunc_desc(env->prog, func_id, offset))
return 0;
if (tab->nr_descs == MAX_KFUNC_DESCS) {
@@ -1721,20 +1864,20 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
return -E2BIG;
}
func = btf_type_by_id(btf_vmlinux, func_id);
func = btf_type_by_id(desc_btf, func_id);
if (!func || !btf_type_is_func(func)) {
verbose(env, "kernel btf_id %u is not a function\n",
func_id);
return -EINVAL;
}
func_proto = btf_type_by_id(btf_vmlinux, func->type);
func_proto = btf_type_by_id(desc_btf, func->type);
if (!func_proto || !btf_type_is_func_proto(func_proto)) {
verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
func_id);
return -EINVAL;
}
func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
func_name = btf_name_by_offset(desc_btf, func->name_off);
addr = kallsyms_lookup_name(func_name);
if (!addr) {
verbose(env, "cannot find address for kernel function %s\n",
@@ -1745,12 +1888,13 @@ static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
desc = &tab->descs[tab->nr_descs++];
desc->func_id = func_id;
desc->imm = BPF_CALL_IMM(addr);
err = btf_distill_func_proto(&env->log, btf_vmlinux,
desc->offset = offset;
err = btf_distill_func_proto(&env->log, desc_btf,
func_proto, func_name,
&desc->func_model);
if (!err)
sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
kfunc_desc_cmp_by_id, NULL);
kfunc_desc_cmp_by_id_off, NULL);
return err;
}
@@ -1829,7 +1973,7 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
} else if (bpf_pseudo_call(insn)) {
ret = add_subprog(env, i + insn->imm + 1);
} else {
ret = add_kfunc_call(env, insn->imm);
ret = add_kfunc_call(env, insn->imm, insn->off);
}
if (ret < 0)
@@ -2166,12 +2310,17 @@ static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
{
const struct btf_type *func;
struct btf *desc_btf;
if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
return NULL;
func = btf_type_by_id(btf_vmlinux, insn->imm);
return btf_name_by_offset(btf_vmlinux, func->name_off);
desc_btf = find_kfunc_desc_btf(data, insn->imm, insn->off, NULL);
if (IS_ERR(desc_btf))
return "<error>";
func = btf_type_by_id(desc_btf, insn->imm);
return btf_name_by_offset(desc_btf, func->name_off);
}
/* For given verifier state backtrack_insn() is called from the last insn to
@@ -6530,23 +6679,33 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
struct bpf_reg_state *regs = cur_regs(env);
const char *func_name, *ptr_type_name;
u32 i, nargs, func_id, ptr_type_id;
struct module *btf_mod = NULL;
const struct btf_param *args;
struct btf *desc_btf;
int err;
/* skip for now, but return error when we find this in fixup_kfunc_call */
if (!insn->imm)
return 0;
desc_btf = find_kfunc_desc_btf(env, insn->imm, insn->off, &btf_mod);
if (IS_ERR(desc_btf))
return PTR_ERR(desc_btf);
func_id = insn->imm;
func = btf_type_by_id(btf_vmlinux, func_id);
func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
func_proto = btf_type_by_id(btf_vmlinux, func->type);
func = btf_type_by_id(desc_btf, func_id);
func_name = btf_name_by_offset(desc_btf, func->name_off);
func_proto = btf_type_by_id(desc_btf, func->type);
if (!env->ops->check_kfunc_call ||
!env->ops->check_kfunc_call(func_id)) {
!env->ops->check_kfunc_call(func_id, btf_mod)) {
verbose(env, "calling kernel function %s is not allowed\n",
func_name);
return -EACCES;
}
/* Check the arguments */
err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs);
if (err)
return err;
@@ -6554,15 +6713,15 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
mark_reg_not_init(env, regs, caller_saved[i]);
/* Check return type */
t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL);
if (btf_type_is_scalar(t)) {
mark_reg_unknown(env, regs, BPF_REG_0);
mark_btf_func_reg_size(env, BPF_REG_0, t->size);
} else if (btf_type_is_ptr(t)) {
ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
ptr_type = btf_type_skip_modifiers(desc_btf, t->type,
&ptr_type_id);
if (!btf_type_is_struct(ptr_type)) {
ptr_type_name = btf_name_by_offset(btf_vmlinux,
ptr_type_name = btf_name_by_offset(desc_btf,
ptr_type->name_off);
verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
func_name, btf_type_str(ptr_type),
@@ -6570,7 +6729,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
return -EINVAL;
}
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].btf = btf_vmlinux;
regs[BPF_REG_0].btf = desc_btf;
regs[BPF_REG_0].type = PTR_TO_BTF_ID;
regs[BPF_REG_0].btf_id = ptr_type_id;
mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
@@ -6581,7 +6740,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
for (i = 0; i < nargs; i++) {
u32 regno = i + 1;
t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
if (btf_type_is_ptr(t))
mark_btf_func_reg_size(env, regno, sizeof(void *));
else
@@ -11121,7 +11280,8 @@ static int do_check(struct bpf_verifier_env *env)
env->jmps_processed++;
if (opcode == BPF_CALL) {
if (BPF_SRC(insn->code) != BPF_K ||
insn->off != 0 ||
(insn->src_reg != BPF_PSEUDO_KFUNC_CALL
&& insn->off != 0) ||
(insn->src_reg != BPF_REG_0 &&
insn->src_reg != BPF_PSEUDO_CALL &&
insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
@@ -12477,6 +12637,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
func[i]->jit_requested = 1;
func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
func[i]->aux->linfo = prog->aux->linfo;
func[i]->aux->nr_linfo = prog->aux->nr_linfo;
func[i]->aux->jited_linfo = prog->aux->jited_linfo;
@@ -12662,10 +12823,15 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env,
{
const struct bpf_kfunc_desc *desc;
if (!insn->imm) {
verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
return -EINVAL;
}
/* insn->imm has the btf func_id. Replace it with
* an address (relative to __bpf_base_call).
*/
desc = find_kfunc_desc(env->prog, insn->imm);
desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
if (!desc) {
verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
insn->imm);
@@ -12946,7 +13112,8 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
insn->imm == BPF_FUNC_map_push_elem ||
insn->imm == BPF_FUNC_map_pop_elem ||
insn->imm == BPF_FUNC_map_peek_elem ||
insn->imm == BPF_FUNC_redirect_map)) {
insn->imm == BPF_FUNC_redirect_map ||
insn->imm == BPF_FUNC_for_each_map_elem)) {
aux = &env->insn_aux_data[i + delta];
if (bpf_map_ptr_poisoned(aux))
goto patch_call_imm;
@@ -12990,6 +13157,11 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
(int (*)(struct bpf_map *map, void *value))NULL));
BUILD_BUG_ON(!__same_type(ops->map_redirect,
(int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL));
BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
(int (*)(struct bpf_map *map,
bpf_callback_t callback_fn,
void *callback_ctx,
u64 flags))NULL));
patch_map_ops_generic:
switch (insn->imm) {
@@ -13014,6 +13186,9 @@ patch_map_ops_generic:
case BPF_FUNC_redirect_map:
insn->imm = BPF_CALL_IMM(ops->map_redirect);
continue;
case BPF_FUNC_for_each_map_elem:
insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
continue;
}
goto patch_call_imm;

120
lib/test_bpf.c
View File

@@ -2134,7 +2134,7 @@ static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self)
* of the immediate value. This is often the case if the native instruction
* immediate field width is narrower than 32 bits.
*/
static int bpf_fill_ld_imm64(struct bpf_test *self)
static int bpf_fill_ld_imm64_magn(struct bpf_test *self)
{
int block = 64; /* Increase for more tests per MSB position */
int len = 3 + 8 * 63 * block * 2;
@@ -2180,6 +2180,88 @@ static int bpf_fill_ld_imm64(struct bpf_test *self)
return 0;
}
/*
* Test the two-instruction 64-bit immediate load operation for different
* combinations of bytes. Each byte in the 64-bit word is constructed as
* (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG.
* All patterns (base1, mask1) and (base2, mask2) bytes are tested.
*/
static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self,
u8 base1, u8 mask1,
u8 base2, u8 mask2)
{
struct bpf_insn *insn;
int len = 3 + 8 * BIT(8);
int pattern, index;
u32 rand = 1;
int i = 0;
insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
if (!insn)
return -ENOMEM;
insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0);
for (pattern = 0; pattern < BIT(8); pattern++) {
u64 imm = 0;
for (index = 0; index < 8; index++) {
int byte;
if (pattern & BIT(index))
byte = (base1 & mask1) | (rand & ~mask1);
else
byte = (base2 & mask2) | (rand & ~mask2);
imm = (imm << 8) | byte;
}
/* Update our LCG */
rand = rand * 1664525 + 1013904223;
/* Perform operation */
i += __bpf_ld_imm64(&insn[i], R1, imm);
/* Load reference */
insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm);
insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32));
insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32);
insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3);
/* Check result */
insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1);
insn[i++] = BPF_EXIT_INSN();
}
insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1);
insn[i++] = BPF_EXIT_INSN();
self->u.ptr.insns = insn;
self->u.ptr.len = len;
BUG_ON(i != len);
return 0;
}
static int bpf_fill_ld_imm64_checker(struct bpf_test *self)
{
return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff);
}
static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self)
{
return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80);
}
static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self)
{
return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff);
}
static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self)
{
return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff);
}
/*
* Exhaustive tests of JMP operations for all combinations of power-of-two
* magnitudes of the operands, both for positive and negative values. The
@@ -12401,14 +12483,46 @@ static struct bpf_test tests[] = {
.fill_helper = bpf_fill_alu32_mod_reg,
.nr_testruns = NR_PATTERN_RUNS,
},
/* LD_IMM64 immediate magnitudes */
/* LD_IMM64 immediate magnitudes and byte patterns */
{
"LD_IMM64: all immediate value magnitudes",
{ },
INTERNAL | FLAG_NO_DATA,
{ },
{ { 0, 1 } },
.fill_helper = bpf_fill_ld_imm64,
.fill_helper = bpf_fill_ld_imm64_magn,
},
{
"LD_IMM64: checker byte patterns",
{ },
INTERNAL | FLAG_NO_DATA,
{ },
{ { 0, 1 } },
.fill_helper = bpf_fill_ld_imm64_checker,
},
{
"LD_IMM64: random positive and zero byte patterns",
{ },
INTERNAL | FLAG_NO_DATA,
{ },
{ { 0, 1 } },
.fill_helper = bpf_fill_ld_imm64_pos_zero,
},
{
"LD_IMM64: random negative and zero byte patterns",
{ },
INTERNAL | FLAG_NO_DATA,
{ },
{ { 0, 1 } },
.fill_helper = bpf_fill_ld_imm64_neg_zero,
},
{
"LD_IMM64: random positive and negative byte patterns",
{ },
INTERNAL | FLAG_NO_DATA,
{ },
{ { 0, 1 } },
.fill_helper = bpf_fill_ld_imm64_pos_neg,
},
/* 64-bit ATOMIC register combinations */
{

7
net/bpf/test_run.c
View File

@@ -2,6 +2,7 @@
/* Copyright (c) 2017 Facebook
*/
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
@@ -241,9 +242,11 @@ BTF_ID(func, bpf_kfunc_call_test2)
BTF_ID(func, bpf_kfunc_call_test3)
BTF_SET_END(test_sk_kfunc_ids)
bool bpf_prog_test_check_kfunc_call(u32 kfunc_id)
bool bpf_prog_test_check_kfunc_call(u32 kfunc_id, struct module *owner)
{
return btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id);
if (btf_id_set_contains(&test_sk_kfunc_ids, kfunc_id))
return true;
return bpf_check_mod_kfunc_call(&prog_test_kfunc_list, kfunc_id, owner);
}
static void *bpf_test_init(const union bpf_attr *kattr, u32 size,

36
net/ipv4/bpf_tcp_ca.c
View File

@@ -223,41 +223,13 @@ BTF_ID(func, tcp_reno_cong_avoid)
BTF_ID(func, tcp_reno_undo_cwnd)
BTF_ID(func, tcp_slow_start)
BTF_ID(func, tcp_cong_avoid_ai)
#ifdef CONFIG_X86
#ifdef CONFIG_DYNAMIC_FTRACE
#if IS_BUILTIN(CONFIG_TCP_CONG_CUBIC)
BTF_ID(func, cubictcp_init)
BTF_ID(func, cubictcp_recalc_ssthresh)
BTF_ID(func, cubictcp_cong_avoid)
BTF_ID(func, cubictcp_state)
BTF_ID(func, cubictcp_cwnd_event)
BTF_ID(func, cubictcp_acked)
#endif
#if IS_BUILTIN(CONFIG_TCP_CONG_DCTCP)
BTF_ID(func, dctcp_init)
BTF_ID(func, dctcp_update_alpha)
BTF_ID(func, dctcp_cwnd_event)
BTF_ID(func, dctcp_ssthresh)
BTF_ID(func, dctcp_cwnd_undo)
BTF_ID(func, dctcp_state)
#endif
#if IS_BUILTIN(CONFIG_TCP_CONG_BBR)
BTF_ID(func, bbr_init)
BTF_ID(func, bbr_main)
BTF_ID(func, bbr_sndbuf_expand)
BTF_ID(func, bbr_undo_cwnd)
BTF_ID(func, bbr_cwnd_event)
BTF_ID(func, bbr_ssthresh)
BTF_ID(func, bbr_min_tso_segs)
BTF_ID(func, bbr_set_state)
#endif
#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_X86 */
BTF_SET_END(bpf_tcp_ca_kfunc_ids)
static bool bpf_tcp_ca_check_kfunc_call(u32 kfunc_btf_id)
static bool bpf_tcp_ca_check_kfunc_call(u32 kfunc_btf_id, struct module *owner)
{
return btf_id_set_contains(&bpf_tcp_ca_kfunc_ids, kfunc_btf_id);
if (btf_id_set_contains(&bpf_tcp_ca_kfunc_ids, kfunc_btf_id))
return true;
return bpf_check_mod_kfunc_call(&bpf_tcp_ca_kfunc_list, kfunc_btf_id, owner);
}
static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {

28
net/ipv4/tcp_bbr.c
View File

@@ -56,6 +56,8 @@
* otherwise TCP stack falls back to an internal pacing using one high
* resolution timer per TCP socket and may use more resources.
*/
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/module.h>
#include <net/tcp.h>
#include <linux/inet_diag.h>
@@ -1152,14 +1154,38 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = {
.set_state = bbr_set_state,
};
BTF_SET_START(tcp_bbr_kfunc_ids)
#ifdef CONFIG_X86
#ifdef CONFIG_DYNAMIC_FTRACE
BTF_ID(func, bbr_init)
BTF_ID(func, bbr_main)
BTF_ID(func, bbr_sndbuf_expand)
BTF_ID(func, bbr_undo_cwnd)
BTF_ID(func, bbr_cwnd_event)
BTF_ID(func, bbr_ssthresh)
BTF_ID(func, bbr_min_tso_segs)
BTF_ID(func, bbr_set_state)
#endif
#endif
BTF_SET_END(tcp_bbr_kfunc_ids)
static DEFINE_KFUNC_BTF_ID_SET(&tcp_bbr_kfunc_ids, tcp_bbr_kfunc_btf_set);
static int __init bbr_register(void)
{
int ret;
BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE);
return tcp_register_congestion_control(&tcp_bbr_cong_ops);
ret = tcp_register_congestion_control(&tcp_bbr_cong_ops);
if (ret)
return ret;
register_kfunc_btf_id_set(&bpf_tcp_ca_kfunc_list, &tcp_bbr_kfunc_btf_set);
return 0;
}
static void __exit bbr_unregister(void)
{
unregister_kfunc_btf_id_set(&bpf_tcp_ca_kfunc_list, &tcp_bbr_kfunc_btf_set);
tcp_unregister_congestion_control(&tcp_bbr_cong_ops);
}

26
net/ipv4/tcp_cubic.c
View File

@@ -25,6 +25,8 @@
*/
#include <linux/mm.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/module.h>
#include <linux/math64.h>
#include <net/tcp.h>
@@ -482,8 +484,25 @@ static struct tcp_congestion_ops cubictcp __read_mostly = {
.name = "cubic",
};
BTF_SET_START(tcp_cubic_kfunc_ids)
#ifdef CONFIG_X86
#ifdef CONFIG_DYNAMIC_FTRACE
BTF_ID(func, cubictcp_init)
BTF_ID(func, cubictcp_recalc_ssthresh)
BTF_ID(func, cubictcp_cong_avoid)
BTF_ID(func, cubictcp_state)
BTF_ID(func, cubictcp_cwnd_event)
BTF_ID(func, cubictcp_acked)
#endif
#endif
BTF_SET_END(tcp_cubic_kfunc_ids)
static DEFINE_KFUNC_BTF_ID_SET(&tcp_cubic_kfunc_ids, tcp_cubic_kfunc_btf_set);
static int __init cubictcp_register(void)
{
int ret;
BUILD_BUG_ON(sizeof(struct bictcp) > ICSK_CA_PRIV_SIZE);
/* Precompute a bunch of the scaling factors that are used per-packet
@@ -514,11 +533,16 @@ static int __init cubictcp_register(void)
/* divide by bic_scale and by constant Srtt (100ms) */
do_div(cube_factor, bic_scale * 10);
return tcp_register_congestion_control(&cubictcp);
ret = tcp_register_congestion_control(&cubictcp);
if (ret)
return ret;
register_kfunc_btf_id_set(&bpf_tcp_ca_kfunc_list, &tcp_cubic_kfunc_btf_set);
return 0;
}
static void __exit cubictcp_unregister(void)
{
unregister_kfunc_btf_id_set(&bpf_tcp_ca_kfunc_list, &tcp_cubic_kfunc_btf_set);
tcp_unregister_congestion_control(&cubictcp);
}

26
net/ipv4/tcp_dctcp.c
View File

@@ -36,6 +36,8 @@
* Glenn Judd <glenn.judd@morganstanley.com>
*/
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <net/tcp.h>
@@ -236,14 +238,36 @@ static struct tcp_congestion_ops dctcp_reno __read_mostly = {
.name = "dctcp-reno",
};
BTF_SET_START(tcp_dctcp_kfunc_ids)
#ifdef CONFIG_X86
#ifdef CONFIG_DYNAMIC_FTRACE
BTF_ID(func, dctcp_init)
BTF_ID(func, dctcp_update_alpha)
BTF_ID(func, dctcp_cwnd_event)
BTF_ID(func, dctcp_ssthresh)
BTF_ID(func, dctcp_cwnd_undo)
BTF_ID(func, dctcp_state)
#endif
#endif
BTF_SET_END(tcp_dctcp_kfunc_ids)
static DEFINE_KFUNC_BTF_ID_SET(&tcp_dctcp_kfunc_ids, tcp_dctcp_kfunc_btf_set);
static int __init dctcp_register(void)
{
int ret;
BUILD_BUG_ON(sizeof(struct dctcp) > ICSK_CA_PRIV_SIZE);
return tcp_register_congestion_control(&dctcp);
ret = tcp_register_congestion_control(&dctcp);
if (ret)
return ret;
register_kfunc_btf_id_set(&bpf_tcp_ca_kfunc_list, &tcp_dctcp_kfunc_btf_set);
return 0;
}
static void __exit dctcp_unregister(void)
{
unregister_kfunc_btf_id_set(&bpf_tcp_ca_kfunc_list, &tcp_dctcp_kfunc_btf_set);
tcp_unregister_congestion_control(&dctcp);
}

4
samples/bpf/.gitignore vendored
View File

@@ -57,3 +57,7 @@ testfile.img
hbm_out.log
iperf.*
*.out
*.skel.h
/vmlinux.h
/bpftool/
/libbpf/

44
samples/bpf/Makefile
View File

@@ -3,6 +3,8 @@
BPF_SAMPLES_PATH ?= $(abspath $(srctree)/$(src))
TOOLS_PATH := $(BPF_SAMPLES_PATH)/../../tools
pound := \#
# List of programs to build
tprogs-y := test_lru_dist
tprogs-y += sock_example
@@ -59,7 +61,11 @@ tprogs-y += xdp_redirect
tprogs-y += xdp_monitor
# Libbpf dependencies
LIBBPF = $(TOOLS_PATH)/lib/bpf/libbpf.a
LIBBPF_SRC = $(TOOLS_PATH)/lib/bpf
LIBBPF_OUTPUT = $(abspath $(BPF_SAMPLES_PATH))/libbpf
LIBBPF_DESTDIR = $(LIBBPF_OUTPUT)
LIBBPF_INCLUDE = $(LIBBPF_DESTDIR)/include
LIBBPF = $(LIBBPF_OUTPUT)/libbpf.a
CGROUP_HELPERS := ../../tools/testing/selftests/bpf/cgroup_helpers.o
TRACE_HELPERS := ../../tools/testing/selftests/bpf/trace_helpers.o
@@ -198,7 +204,7 @@ TPROGS_CFLAGS += -Wstrict-prototypes
TPROGS_CFLAGS += -I$(objtree)/usr/include
TPROGS_CFLAGS += -I$(srctree)/tools/testing/selftests/bpf/
TPROGS_CFLAGS += -I$(srctree)/tools/lib/
TPROGS_CFLAGS += -I$(LIBBPF_INCLUDE)
TPROGS_CFLAGS += -I$(srctree)/tools/include
TPROGS_CFLAGS += -I$(srctree)/tools/perf
TPROGS_CFLAGS += -DHAVE_ATTR_TEST=0
@@ -232,7 +238,7 @@ endif
# Don't evaluate probes and warnings if we need to run make recursively
ifneq ($(src),)
HDR_PROBE := $(shell printf "\#include <linux/types.h>\n struct list_head { int a; }; int main() { return 0; }" | \
HDR_PROBE := $(shell printf "$(pound)include <linux/types.h>\n struct list_head { int a; }; int main() { return 0; }" | \
$(CC) $(TPROGS_CFLAGS) $(TPROGS_LDFLAGS) -x c - \
-o /dev/null 2>/dev/null && echo okay)
@@ -268,16 +274,27 @@ all:
clean:
$(MAKE) -C ../../ M=$(CURDIR) clean
@find $(CURDIR) -type f -name '*~' -delete
@$(RM) -r $(CURDIR)/libbpf $(CURDIR)/bpftool
$(LIBBPF): FORCE
$(LIBBPF): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUTPUT)
# Fix up variables inherited from Kbuild that tools/ build system won't like
$(MAKE) -C $(dir $@) RM='rm -rf' EXTRA_CFLAGS="$(TPROGS_CFLAGS)" \
LDFLAGS=$(TPROGS_LDFLAGS) srctree=$(BPF_SAMPLES_PATH)/../../ O=
$(MAKE) -C $(LIBBPF_SRC) RM='rm -rf' EXTRA_CFLAGS="$(TPROGS_CFLAGS)" \
LDFLAGS=$(TPROGS_LDFLAGS) srctree=$(BPF_SAMPLES_PATH)/../../ \
O= OUTPUT=$(LIBBPF_OUTPUT)/ DESTDIR=$(LIBBPF_DESTDIR) prefix= \
$@ install_headers
BPFTOOLDIR := $(TOOLS_PATH)/bpf/bpftool
BPFTOOL := $(BPFTOOLDIR)/bpftool
$(BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile)
$(MAKE) -C $(BPFTOOLDIR) srctree=$(BPF_SAMPLES_PATH)/../../
BPFTOOL_OUTPUT := $(abspath $(BPF_SAMPLES_PATH))/bpftool
BPFTOOL := $(BPFTOOL_OUTPUT)/bpftool
$(BPFTOOL): $(LIBBPF) $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) | $(BPFTOOL_OUTPUT)
$(MAKE) -C $(BPFTOOLDIR) srctree=$(BPF_SAMPLES_PATH)/../../ \
OUTPUT=$(BPFTOOL_OUTPUT)/ \
LIBBPF_OUTPUT=$(LIBBPF_OUTPUT)/ \
LIBBPF_DESTDIR=$(LIBBPF_DESTDIR)/
$(LIBBPF_OUTPUT) $(BPFTOOL_OUTPUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $@
$(obj)/syscall_nrs.h: $(obj)/syscall_nrs.s FORCE
$(call filechk,offsets,__SYSCALL_NRS_H__)
@@ -309,6 +326,11 @@ verify_target_bpf: verify_cmds
$(BPF_SAMPLES_PATH)/*.c: verify_target_bpf $(LIBBPF)
$(src)/*.c: verify_target_bpf $(LIBBPF)
libbpf_hdrs: $(LIBBPF)
$(obj)/$(TRACE_HELPERS): | libbpf_hdrs
.PHONY: libbpf_hdrs
$(obj)/xdp_redirect_cpu_user.o: $(obj)/xdp_redirect_cpu.skel.h
$(obj)/xdp_redirect_map_multi_user.o: $(obj)/xdp_redirect_map_multi.skel.h
$(obj)/xdp_redirect_map_user.o: $(obj)/xdp_redirect_map.skel.h
@@ -366,7 +388,7 @@ $(obj)/%.bpf.o: $(src)/%.bpf.c $(obj)/vmlinux.h $(src)/xdp_sample.bpf.h $(src)/x
$(Q)$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(SRCARCH) \
-Wno-compare-distinct-pointer-types -I$(srctree)/include \
-I$(srctree)/samples/bpf -I$(srctree)/tools/include \
-I$(srctree)/tools/lib $(CLANG_SYS_INCLUDES) \
-I$(LIBBPF_INCLUDE) $(CLANG_SYS_INCLUDES) \
-c $(filter %.bpf.c,$^) -o $@
LINKED_SKELS := xdp_redirect_cpu.skel.h xdp_redirect_map_multi.skel.h \
@@ -403,7 +425,7 @@ $(obj)/%.o: $(src)/%.c
@echo " CLANG-bpf " $@
$(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \
-I$(obj) -I$(srctree)/tools/testing/selftests/bpf/ \
-I$(srctree)/tools/lib/ \
-I$(LIBBPF_INCLUDE) \
-D__KERNEL__ -D__BPF_TRACING__ -Wno-unused-value -Wno-pointer-sign \
-D__TARGET_ARCH_$(SRCARCH) -Wno-compare-distinct-pointer-types \
-Wno-gnu-variable-sized-type-not-at-end \

2
samples/bpf/xdp1_user.c
View File

@@ -128,7 +128,7 @@ int main(int argc, char **argv)
if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))
return 1;
map = bpf_map__next(NULL, obj);
map = bpf_object__next_map(obj, NULL);
if (!map) {
printf("finding a map in obj file failed\n");
return 1;

2
samples/bpf/xdp_sample_pkts_user.c
View File

@@ -154,7 +154,7 @@ int main(int argc, char **argv)
return 1;
}
map = bpf_map__next(NULL, obj);
map = bpf_object__next_map(obj, NULL);
if (!map) {
printf("finding a map in obj file failed\n");
return 1;

1
scripts/Makefile.modfinal
View File

@@ -41,6 +41,7 @@ quiet_cmd_btf_ko = BTF [M] $@
cmd_btf_ko = \
if [ -f vmlinux ]; then \
LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J --btf_base vmlinux $@; \
$(RESOLVE_BTFIDS) -b vmlinux $@; \
else \
printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \
fi;

39
tools/bpf/bpftool/Makefile
View File

@@ -17,19 +17,23 @@ endif
BPF_DIR = $(srctree)/tools/lib/bpf/
ifneq ($(OUTPUT),)
LIBBPF_OUTPUT = $(OUTPUT)/libbpf/
LIBBPF_PATH = $(LIBBPF_OUTPUT)
BOOTSTRAP_OUTPUT = $(OUTPUT)/bootstrap/
_OUTPUT := $(OUTPUT)
else
LIBBPF_OUTPUT =
LIBBPF_PATH = $(BPF_DIR)
BOOTSTRAP_OUTPUT = $(CURDIR)/bootstrap/
_OUTPUT := $(CURDIR)
endif
BOOTSTRAP_OUTPUT := $(_OUTPUT)/bootstrap/
LIBBPF_OUTPUT := $(_OUTPUT)/libbpf/
LIBBPF_DESTDIR := $(LIBBPF_OUTPUT)
LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)/include
LIBBPF = $(LIBBPF_PATH)libbpf.a
LIBBPF = $(LIBBPF_OUTPUT)libbpf.a
LIBBPF_BOOTSTRAP_OUTPUT = $(BOOTSTRAP_OUTPUT)libbpf/
LIBBPF_BOOTSTRAP = $(LIBBPF_BOOTSTRAP_OUTPUT)libbpf.a
# We need to copy nlattr.h which is not otherwise exported by libbpf, but still
# required by bpftool.
LIBBPF_INTERNAL_HDRS := nlattr.h
ifeq ($(BPFTOOL_VERSION),)
BPFTOOL_VERSION := $(shell make -rR --no-print-directory -sC ../../.. kernelversion)
endif
@@ -38,7 +42,13 @@ $(LIBBPF_OUTPUT) $(BOOTSTRAP_OUTPUT) $(LIBBPF_BOOTSTRAP_OUTPUT):
$(QUIET_MKDIR)mkdir -p $@
$(LIBBPF): FORCE | $(LIBBPF_OUTPUT)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_OUTPUT) $(LIBBPF_OUTPUT)libbpf.a
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_OUTPUT) \
DESTDIR=$(LIBBPF_DESTDIR) prefix= $(LIBBPF) install_headers
$(LIBBPF_INCLUDE)/bpf/$(LIBBPF_INTERNAL_HDRS): \
$(addprefix $(BPF_DIR),$(LIBBPF_INTERNAL_HDRS)) $(LIBBPF)
$(call QUIET_INSTALL, bpf/$(notdir $@))
$(Q)install -m 644 -t $(LIBBPF_INCLUDE)/bpf/ $(BPF_DIR)$(notdir $@)
$(LIBBPF_BOOTSTRAP): FORCE | $(LIBBPF_BOOTSTRAP_OUTPUT)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \
@@ -60,10 +70,10 @@ CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers
CFLAGS += $(filter-out -Wswitch-enum -Wnested-externs,$(EXTRA_WARNINGS))
CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ \
-I$(if $(OUTPUT),$(OUTPUT),.) \
-I$(LIBBPF_INCLUDE) \
-I$(srctree)/kernel/bpf/ \
-I$(srctree)/tools/include \
-I$(srctree)/tools/include/uapi \
-I$(srctree)/tools/lib \
-I$(srctree)/tools/perf
CFLAGS += -DBPFTOOL_VERSION='"$(BPFTOOL_VERSION)"'
ifneq ($(EXTRA_CFLAGS),)
@@ -140,7 +150,7 @@ BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o g
$(BOOTSTRAP_OBJS): $(LIBBPF_BOOTSTRAP)
OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
$(OBJS): $(LIBBPF)
$(OBJS): $(LIBBPF) $(LIBBPF_INCLUDE)/bpf/$(LIBBPF_INTERNAL_HDRS)
VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \
$(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \
@@ -167,8 +177,7 @@ $(OUTPUT)%.bpf.o: skeleton/%.bpf.c $(OUTPUT)vmlinux.h $(LIBBPF)
$(QUIET_CLANG)$(CLANG) \
-I$(if $(OUTPUT),$(OUTPUT),.) \
-I$(srctree)/tools/include/uapi/ \
-I$(LIBBPF_PATH) \
-I$(srctree)/tools/lib \
-I$(LIBBPF_INCLUDE) \
-g -O2 -Wall -target bpf -c $< -o $@ && $(LLVM_STRIP) -g $@
$(OUTPUT)%.skel.h: $(OUTPUT)%.bpf.o $(BPFTOOL_BOOTSTRAP)
@@ -217,10 +226,12 @@ clean: $(LIBBPF)-clean $(LIBBPF_BOOTSTRAP)-clean feature-detect-clean
$(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpftool
$(Q)$(RM) -r -- $(OUTPUT)feature/
install: $(OUTPUT)bpftool
install-bin: $(OUTPUT)bpftool
$(call QUIET_INSTALL, bpftool)
$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/sbin
$(Q)$(INSTALL) $(OUTPUT)bpftool $(DESTDIR)$(prefix)/sbin/bpftool
install: install-bin
$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(bash_compdir)
$(Q)$(INSTALL) -m 0644 bash-completion/bpftool $(DESTDIR)$(bash_compdir)
@@ -247,6 +258,6 @@ zdep:
@if [ "$(feature-zlib)" != "1" ]; then echo "No zlib found"; exit 1 ; fi
.SECONDARY:
.PHONY: all FORCE clean install uninstall zdep
.PHONY: all FORCE clean install-bin install uninstall zdep
.PHONY: doc doc-clean doc-install doc-uninstall
.DEFAULT_GOAL := all

1
tools/bpf/bpftool/gen.c
View File

@@ -18,7 +18,6 @@
#include <sys/stat.h>
#include <sys/mman.h>
#include <bpf/btf.h>
#include <bpf/bpf_gen_internal.h>
#include "json_writer.h"
#include "main.h"

2
tools/bpf/bpftool/iter.c
View File

@@ -57,7 +57,7 @@ static int do_pin(int argc, char **argv)
goto close_obj;
}
prog = bpf_program__next(NULL, obj);
prog = bpf_object__next_program(obj, NULL);
if (!prog) {
p_err("can't find bpf program in objfile %s", objfile);
goto close_obj;

3
tools/bpf/bpftool/prog.c
View File

@@ -25,7 +25,6 @@
#include <bpf/bpf.h>
#include <bpf/btf.h>
#include <bpf/libbpf.h>
#include <bpf/bpf_gen_internal.h>
#include <bpf/skel_internal.h>
#include "cfg.h"
@@ -1601,7 +1600,7 @@ static int load_with_options(int argc, char **argv, bool first_prog_only)
goto err_close_obj;
if (first_prog_only) {
prog = bpf_program__next(NULL, obj);
prog = bpf_object__next_program(obj, NULL);
if (!prog) {
p_err("object file doesn't contain any bpf program");
goto err_close_obj;

16
tools/bpf/resolve_btfids/Makefile
View File

@@ -29,25 +29,30 @@ BPFOBJ := $(OUTPUT)/libbpf/libbpf.a
LIBBPF_OUT := $(abspath $(dir $(BPFOBJ)))/
SUBCMDOBJ := $(OUTPUT)/libsubcmd/libsubcmd.a
LIBBPF_DESTDIR := $(LIBBPF_OUT)
LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include
BINARY := $(OUTPUT)/resolve_btfids
BINARY_IN := $(BINARY)-in.o
all: $(BINARY)
$(OUTPUT) $(OUTPUT)/libbpf $(OUTPUT)/libsubcmd:
$(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
$(call msg,MKDIR,,$@)
$(Q)mkdir -p $(@)
$(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT) $(abspath $@)
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT) \
DESTDIR=$(LIBBPF_DESTDIR) prefix= \
$(abspath $@) install_headers
CFLAGS := -g \
-I$(srctree)/tools/include \
-I$(srctree)/tools/include/uapi \
-I$(LIBBPF_SRC) \
-I$(LIBBPF_INCLUDE) \
-I$(SUBCMD_SRC)
LIBS = -lelf -lz
@@ -65,7 +70,8 @@ $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
clean_objects := $(wildcard $(OUTPUT)/*.o \
$(OUTPUT)/.*.o.cmd \
$(OUTPUT)/.*.o.d \
$(OUTPUT)/libbpf \
$(LIBBPF_OUT) \
$(LIBBPF_DESTDIR) \
$(OUTPUT)/libsubcmd \
$(OUTPUT)/resolve_btfids)

32
tools/bpf/resolve_btfids/main.c
View File

@@ -60,8 +60,8 @@
#include <linux/rbtree.h>
#include <linux/zalloc.h>
#include <linux/err.h>
#include <btf.h>
#include <libbpf.h>
#include <bpf/btf.h>
#include <bpf/libbpf.h>
#include <parse-options.h>
#define BTF_IDS_SECTION ".BTF_ids"
@@ -89,6 +89,7 @@ struct btf_id {
struct object {
const char *path;
const char *btf;
const char *base_btf_path;
struct {
int fd;
@@ -477,16 +478,27 @@ static int symbols_resolve(struct object *obj)
int nr_structs = obj->nr_structs;
int nr_unions = obj->nr_unions;
int nr_funcs = obj->nr_funcs;
struct btf *base_btf = NULL;
int err, type_id;
struct btf *btf;
__u32 nr_types;
btf = btf__parse(obj->btf ?: obj->path, NULL);
if (obj->base_btf_path) {
base_btf = btf__parse(obj->base_btf_path, NULL);
err = libbpf_get_error(base_btf);
if (err) {
pr_err("FAILED: load base BTF from %s: %s\n",
obj->base_btf_path, strerror(-err));
return -1;
}
}
btf = btf__parse_split(obj->btf ?: obj->path, base_btf);
err = libbpf_get_error(btf);
if (err) {
pr_err("FAILED: load BTF from %s: %s\n",
obj->btf ?: obj->path, strerror(-err));
return -1;
goto out;
}
err = -1;
@@ -545,6 +557,7 @@ static int symbols_resolve(struct object *obj)
err = 0;
out:
btf__free(base_btf);
btf__free(btf);
return err;
}
@@ -678,7 +691,6 @@ static const char * const resolve_btfids_usage[] = {
int main(int argc, const char **argv)
{
bool no_fail = false;
struct object obj = {
.efile = {
.idlist_shndx = -1,
@@ -695,8 +707,8 @@ int main(int argc, const char **argv)
"be more verbose (show errors, etc)"),
OPT_STRING(0, "btf", &obj.btf, "BTF data",
"BTF data"),
OPT_BOOLEAN(0, "no-fail", &no_fail,
"do not fail if " BTF_IDS_SECTION " section is not found"),
OPT_STRING('b', "btf_base", &obj.base_btf_path, "file",
"path of file providing base BTF"),
OPT_END()
};
int err = -1;
@@ -717,10 +729,8 @@ int main(int argc, const char **argv)
*/
if (obj.efile.idlist_shndx == -1 ||
obj.efile.symbols_shndx == -1) {
if (no_fail)
return 0;
pr_err("FAILED to find needed sections\n");
return -1;
pr_debug("Cannot find .BTF_ids or symbols sections, nothing to do\n");
return 0;
}
if (symbols_collect(&obj))

22
tools/bpf/runqslower/Makefile
View File

@@ -9,9 +9,9 @@ BPFTOOL ?= $(DEFAULT_BPFTOOL)
LIBBPF_SRC := $(abspath ../../lib/bpf)
BPFOBJ_OUTPUT := $(OUTPUT)libbpf/
BPFOBJ := $(BPFOBJ_OUTPUT)libbpf.a
BPF_INCLUDE := $(BPFOBJ_OUTPUT)
INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../lib) \
-I$(abspath ../../include/uapi)
BPF_DESTDIR := $(BPFOBJ_OUTPUT)
BPF_INCLUDE := $(BPF_DESTDIR)/include
INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../include/uapi)
CFLAGS := -g -Wall
# Try to detect best kernel BTF source
@@ -33,7 +33,7 @@ endif
.DELETE_ON_ERROR:
.PHONY: all clean runqslower
.PHONY: all clean runqslower libbpf_hdrs
all: runqslower
runqslower: $(OUTPUT)/runqslower
@@ -46,13 +46,15 @@ clean:
$(Q)$(RM) $(OUTPUT)runqslower
$(Q)$(RM) -r .output
libbpf_hdrs: $(BPFOBJ)
$(OUTPUT)/runqslower: $(OUTPUT)/runqslower.o $(BPFOBJ)
$(QUIET_LINK)$(CC) $(CFLAGS) $^ -lelf -lz -o $@
$(OUTPUT)/runqslower.o: runqslower.h $(OUTPUT)/runqslower.skel.h \
$(OUTPUT)/runqslower.bpf.o
$(OUTPUT)/runqslower.bpf.o | libbpf_hdrs
$(OUTPUT)/runqslower.bpf.o: $(OUTPUT)/vmlinux.h runqslower.h
$(OUTPUT)/runqslower.bpf.o: $(OUTPUT)/vmlinux.h runqslower.h | libbpf_hdrs
$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(BPFTOOL)
$(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@
@@ -81,8 +83,10 @@ else
endif
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(BPFOBJ_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(BPFOBJ_OUTPUT) $@
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(BPFOBJ_OUTPUT) \
DESTDIR=$(BPFOBJ_OUTPUT) prefix= $(abspath $@) install_headers
$(DEFAULT_BPFTOOL): | $(BPFTOOL_OUTPUT)
$(DEFAULT_BPFTOOL): $(BPFOBJ) | $(BPFTOOL_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) \
CC=$(HOSTCC) LD=$(HOSTLD)
LIBBPF_OUTPUT=$(BPFOBJ_OUTPUT) \
LIBBPF_DESTDIR=$(BPF_DESTDIR) CC=$(HOSTCC) LD=$(HOSTLD)

29
tools/lib/bpf/Makefile
View File

@@ -208,8 +208,8 @@ check_abi: $(OUTPUT)libbpf.so $(VERSION_SCRIPT)
exit 1; \
fi
HDR_MAJ_VERSION := $(shell grep -oE '^\#define LIBBPF_MAJOR_VERSION ([0-9]+)$$' libbpf_version.h | cut -d' ' -f3)
HDR_MIN_VERSION := $(shell grep -oE '^\#define LIBBPF_MINOR_VERSION ([0-9]+)$$' libbpf_version.h | cut -d' ' -f3)
HDR_MAJ_VERSION := $(shell grep -oE '^$(pound)define LIBBPF_MAJOR_VERSION ([0-9]+)$$' libbpf_version.h | cut -d' ' -f3)
HDR_MIN_VERSION := $(shell grep -oE '^$(pound)define LIBBPF_MINOR_VERSION ([0-9]+)$$' libbpf_version.h | cut -d' ' -f3)
check_version: $(VERSION_SCRIPT) libbpf_version.h
@if [ "$(HDR_MAJ_VERSION)" != "$(LIBBPF_MAJOR_VERSION)" ]; then \
@@ -241,15 +241,24 @@ install_lib: all_cmd
$(call do_install_mkdir,$(libdir_SQ)); \
cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ)
INSTALL_HEADERS = bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \
bpf_helpers.h $(BPF_GENERATED) bpf_tracing.h \
bpf_endian.h bpf_core_read.h skel_internal.h \
libbpf_version.h
SRC_HDRS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \
bpf_helpers.h bpf_tracing.h bpf_endian.h bpf_core_read.h \
skel_internal.h libbpf_version.h
GEN_HDRS := $(BPF_GENERATED)
install_headers: $(BPF_GENERATED)
$(call QUIET_INSTALL, headers) \
$(foreach hdr,$(INSTALL_HEADERS), \
$(call do_install,$(hdr),$(prefix)/include/bpf,644);)
INSTALL_PFX := $(DESTDIR)$(prefix)/include/bpf
INSTALL_SRC_HDRS := $(addprefix $(INSTALL_PFX)/, $(SRC_HDRS))
INSTALL_GEN_HDRS := $(addprefix $(INSTALL_PFX)/, $(notdir $(GEN_HDRS)))
$(INSTALL_SRC_HDRS): $(INSTALL_PFX)/%.h: %.h
$(call QUIET_INSTALL, $@) \
$(call do_install,$<,$(prefix)/include/bpf,644)
$(INSTALL_GEN_HDRS): $(INSTALL_PFX)/%.h: $(OUTPUT)%.h
$(call QUIET_INSTALL, $@) \
$(call do_install,$<,$(prefix)/include/bpf,644)
install_headers: $(BPF_GENERATED) $(INSTALL_SRC_HDRS) $(INSTALL_GEN_HDRS)
install_pkgconfig: $(PC_FILE)
$(call QUIET_INSTALL, $(PC_FILE)) \

1
tools/lib/bpf/bpf.c
View File

@@ -264,6 +264,7 @@ int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr)
attr.line_info_rec_size = load_attr->line_info_rec_size;
attr.line_info_cnt = load_attr->line_info_cnt;
attr.line_info = ptr_to_u64(load_attr->line_info);
attr.fd_array = ptr_to_u64(load_attr->fd_array);
if (load_attr->name)
memcpy(attr.prog_name, load_attr->name,

16
tools/lib/bpf/bpf_gen_internal.h
View File

@@ -7,6 +7,15 @@ struct ksym_relo_desc {
const char *name;
int kind;
int insn_idx;
bool is_weak;
};
struct ksym_desc {
const char *name;
int ref;
int kind;
int off;
int insn;
};
struct bpf_gen {
@@ -24,6 +33,10 @@ struct bpf_gen {
int relo_cnt;
char attach_target[128];
int attach_kind;
struct ksym_desc *ksyms;
__u32 nr_ksyms;
int fd_array;
int nr_fd_array;
};
void bpf_gen__init(struct bpf_gen *gen, int log_level);
@@ -36,6 +49,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, struct bpf_prog_load_params *load_a
void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size);
void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx);
void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type);
void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind, int insn_idx);
void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, int kind,
int insn_idx);
#endif

132
tools/lib/bpf/btf.c
View File

@@ -189,12 +189,17 @@ int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_
return 0;
}
static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt)
{
return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
btf->nr_types, BTF_MAX_NR_TYPES, add_cnt);
}
static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
{
__u32 *p;
p = libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
btf->nr_types, BTF_MAX_NR_TYPES, 1);
p = btf_add_type_offs_mem(btf, 1);
if (!p)
return -ENOMEM;
@@ -695,15 +700,15 @@ __s32 btf__find_by_name(const struct btf *btf, const char *type_name)
return libbpf_err(-ENOENT);
}
__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
__u32 kind)
static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id,
const char *type_name, __u32 kind)
{
__u32 i, nr_types = btf__get_nr_types(btf);
if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
return 0;
for (i = 1; i <= nr_types; i++) {
for (i = start_id; i <= nr_types; i++) {
const struct btf_type *t = btf__type_by_id(btf, i);
const char *name;
@@ -717,6 +722,18 @@ __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
return libbpf_err(-ENOENT);
}
__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
__u32 kind)
{
return btf_find_by_name_kind(btf, btf->start_id, type_name, kind);
}
__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
__u32 kind)
{
return btf_find_by_name_kind(btf, 1, type_name, kind);
}
static bool btf_is_modifiable(const struct btf *btf)
{
return (void *)btf->hdr != btf->raw_data;
@@ -1691,6 +1708,111 @@ int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_t
return btf_commit_type(btf, sz);
}
static int btf_rewrite_type_ids(__u32 *type_id, void *ctx)
{
struct btf *btf = ctx;
if (!*type_id) /* nothing to do for VOID references */
return 0;
/* we haven't updated btf's type count yet, so
* btf->start_id + btf->nr_types - 1 is the type ID offset we should
* add to all newly added BTF types
*/
*type_id += btf->start_id + btf->nr_types - 1;
return 0;
}
int btf__add_btf(struct btf *btf, const struct btf *src_btf)
{
struct btf_pipe p = { .src = src_btf, .dst = btf };
int data_sz, sz, cnt, i, err, old_strs_len;
__u32 *off;
void *t;
/* appending split BTF isn't supported yet */
if (src_btf->base_btf)
return libbpf_err(-ENOTSUP);
/* deconstruct BTF, if necessary, and invalidate raw_data */
if (btf_ensure_modifiable(btf))
return libbpf_err(-ENOMEM);
/* remember original strings section size if we have to roll back
* partial strings section changes
*/
old_strs_len = btf->hdr->str_len;
data_sz = src_btf->hdr->type_len;
cnt = btf__get_nr_types(src_btf);
/* pre-allocate enough memory for new types */
t = btf_add_type_mem(btf, data_sz);
if (!t)
return libbpf_err(-ENOMEM);
/* pre-allocate enough memory for type offset index for new types */
off = btf_add_type_offs_mem(btf, cnt);
if (!off)
return libbpf_err(-ENOMEM);
/* bulk copy types data for all types from src_btf */
memcpy(t, src_btf->types_data, data_sz);
for (i = 0; i < cnt; i++) {
sz = btf_type_size(t);
if (sz < 0) {
/* unlikely, has to be corrupted src_btf */
err = sz;
goto err_out;
}
/* fill out type ID to type offset mapping for lookups by type ID */
*off = t - btf->types_data;
/* add, dedup, and remap strings referenced by this BTF type */
err = btf_type_visit_str_offs(t, btf_rewrite_str, &p);
if (err)
goto err_out;
/* remap all type IDs referenced from this BTF type */
err = btf_type_visit_type_ids(t, btf_rewrite_type_ids, btf);
if (err)
goto err_out;
/* go to next type data and type offset index entry */
t += sz;
off++;
}
/* Up until now any of the copied type data was effectively invisible,
* so if we exited early before this point due to error, BTF would be
* effectively unmodified. There would be extra internal memory
* pre-allocated, but it would not be available for querying. But now
* that we've copied and rewritten all the data successfully, we can
* update type count and various internal offsets and sizes to
* "commit" the changes and made them visible to the outside world.
*/
btf->hdr->type_len += data_sz;
btf->hdr->str_off += data_sz;
btf->nr_types += cnt;
/* return type ID of the first added BTF type */
return btf->start_id + btf->nr_types - cnt;
err_out:
/* zero out preallocated memory as if it was just allocated with
* libbpf_add_mem()
*/
memset(btf->types_data + btf->hdr->type_len, 0, data_sz);
memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len);
/* and now restore original strings section size; types data size
* wasn't modified, so doesn't need restoring, see big comment above */
btf->hdr->str_len = old_strs_len;
return libbpf_err(err);
}
/*
* Append new BTF_KIND_INT type with:
* - *name* - non-empty, non-NULL type name;

22
tools/lib/bpf/btf.h
View File

@@ -173,6 +173,28 @@ LIBBPF_API int btf__find_str(struct btf *btf, const char *s);
LIBBPF_API int btf__add_str(struct btf *btf, const char *s);
LIBBPF_API int btf__add_type(struct btf *btf, const struct btf *src_btf,
const struct btf_type *src_type);
/**
* @brief **btf__add_btf()** appends all the BTF types from *src_btf* into *btf*
* @param btf BTF object which all the BTF types and strings are added to
* @param src_btf BTF object which all BTF types and referenced strings are copied from
* @return BTF type ID of the first appended BTF type, or negative error code
*
* **btf__add_btf()** can be used to simply and efficiently append the entire
* contents of one BTF object to another one. All the BTF type data is copied
* over, all referenced type IDs are adjusted by adding a necessary ID offset.
* Only strings referenced from BTF types are copied over and deduplicated, so
* if there were some unused strings in *src_btf*, those won't be copied over,
* which is consistent with the general string deduplication semantics of BTF
* writing APIs.
*
* If any error is encountered during this process, the contents of *btf* is
* left intact, which means that **btf__add_btf()** follows the transactional
* semantics and the operation as a whole is all-or-nothing.
*
* *src_btf* has to be non-split BTF, as of now copying types from split BTF
* is not supported and will result in -ENOTSUP error code returned.
*/
LIBBPF_API int btf__add_btf(struct btf *btf, const struct btf *src_btf);
LIBBPF_API int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding);
LIBBPF_API int btf__add_float(struct btf *btf, const char *name, size_t byte_sz);

314
tools/lib/bpf/gen_loader.c
View File

@@ -14,8 +14,10 @@
#include "bpf_gen_internal.h"
#include "skel_internal.h"
#define MAX_USED_MAPS 64
#define MAX_USED_PROGS 32
#define MAX_USED_MAPS 64
#define MAX_USED_PROGS 32
#define MAX_KFUNC_DESCS 256
#define MAX_FD_ARRAY_SZ (MAX_USED_PROGS + MAX_KFUNC_DESCS)
/* The following structure describes the stack layout of the loader program.
* In addition R6 contains the pointer to context.
@@ -30,7 +32,6 @@
*/
struct loader_stack {
__u32 btf_fd;
__u32 map_fd[MAX_USED_MAPS];
__u32 prog_fd[MAX_USED_PROGS];
__u32 inner_map_fd;
};
@@ -143,13 +144,49 @@ static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
if (realloc_data_buf(gen, size8))
return 0;
prev = gen->data_cur;
memcpy(gen->data_cur, data, size);
gen->data_cur += size;
memcpy(gen->data_cur, &zero, size8 - size);
gen->data_cur += size8 - size;
if (data) {
memcpy(gen->data_cur, data, size);
memcpy(gen->data_cur + size, &zero, size8 - size);
} else {
memset(gen->data_cur, 0, size8);
}
gen->data_cur += size8;
return prev - gen->data_start;
}
/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
* to start of fd_array. Caller can decide if it is usable or not.
*/
static int add_map_fd(struct bpf_gen *gen)
{
if (!gen->fd_array)
gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
if (gen->nr_maps == MAX_USED_MAPS) {
pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
gen->error = -E2BIG;
return 0;
}
return gen->nr_maps++;
}
static int add_kfunc_btf_fd(struct bpf_gen *gen)
{
int cur;
if (!gen->fd_array)
gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
cur = add_data(gen, NULL, sizeof(int));
return (cur - gen->fd_array) / sizeof(int);
}
return MAX_USED_MAPS + gen->nr_fd_array++;
}
static int blob_fd_array_off(struct bpf_gen *gen, int index)
{
return gen->fd_array + index * sizeof(int);
}
static int insn_bytes_to_bpf_size(__u32 sz)
{
switch (sz) {
@@ -171,14 +208,22 @@ static void emit_rel_store(struct bpf_gen *gen, int off, int data)
emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
}
/* *(u64 *)(blob + off) = (u64)(void *)(%sp + stack_off) */
static void emit_rel_store_sp(struct bpf_gen *gen, int off, int stack_off)
static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
{
emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_10));
emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, stack_off));
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, blob_off));
emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, off));
emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
}
static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
{
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, blob_off));
emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
}
static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
@@ -326,11 +371,11 @@ int bpf_gen__finish(struct bpf_gen *gen)
offsetof(struct bpf_prog_desc, prog_fd), 4,
stack_off(prog_fd[i]));
for (i = 0; i < gen->nr_maps; i++)
move_stack2ctx(gen,
sizeof(struct bpf_loader_ctx) +
sizeof(struct bpf_map_desc) * i +
offsetof(struct bpf_map_desc, map_fd), 4,
stack_off(map_fd[i]));
move_blob2ctx(gen,
sizeof(struct bpf_loader_ctx) +
sizeof(struct bpf_map_desc) * i +
offsetof(struct bpf_map_desc, map_fd), 4,
blob_fd_array_off(gen, i));
emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
emit(gen, BPF_EXIT_INSN());
pr_debug("gen: finish %d\n", gen->error);
@@ -390,7 +435,7 @@ void bpf_gen__map_create(struct bpf_gen *gen,
{
int attr_size = offsetofend(union bpf_attr, btf_vmlinux_value_type_id);
bool close_inner_map_fd = false;
int map_create_attr;
int map_create_attr, idx;
union bpf_attr attr;
memset(&attr, 0, attr_size);
@@ -467,9 +512,11 @@ void bpf_gen__map_create(struct bpf_gen *gen,
gen->error = -EDOM; /* internal bug */
return;
} else {
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
stack_off(map_fd[map_idx])));
gen->nr_maps++;
/* add_map_fd does gen->nr_maps++ */
idx = add_map_fd(gen);
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, blob_fd_array_off(gen, idx)));
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
}
if (close_inner_map_fd)
emit_sys_close_stack(gen, stack_off(inner_map_fd));
@@ -511,8 +558,8 @@ static void emit_find_attach_target(struct bpf_gen *gen)
*/
}
void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind,
int insn_idx)
void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
int kind, int insn_idx)
{
struct ksym_relo_desc *relo;
@@ -524,38 +571,192 @@ void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, int kind,
gen->relos = relo;
relo += gen->relo_cnt;
relo->name = name;
relo->is_weak = is_weak;
relo->kind = kind;
relo->insn_idx = insn_idx;
gen->relo_cnt++;
}
static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
/* returns existing ksym_desc with ref incremented, or inserts a new one */
static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
{
int name, insn, len = strlen(relo->name) + 1;
struct ksym_desc *kdesc;
pr_debug("gen: emit_relo: %s at %d\n", relo->name, relo->insn_idx);
name = add_data(gen, relo->name, len);
for (int i = 0; i < gen->nr_ksyms; i++) {
if (!strcmp(gen->ksyms[i].name, relo->name)) {
gen->ksyms[i].ref++;
return &gen->ksyms[i];
}
}
kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
if (!kdesc) {
gen->error = -ENOMEM;
return NULL;
}
gen->ksyms = kdesc;
kdesc = &gen->ksyms[gen->nr_ksyms++];
kdesc->name = relo->name;
kdesc->kind = relo->kind;
kdesc->ref = 1;
kdesc->off = 0;
kdesc->insn = 0;
return kdesc;
}
/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
* Returns result in BPF_REG_7
*/
static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
{
int name_off, len = strlen(relo->name) + 1;
name_off = add_data(gen, relo->name, len);
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, name));
0, 0, 0, name_off));
emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
}
/* Expects:
* BPF_REG_8 - pointer to instruction
*
* We need to reuse BTF fd for same symbol otherwise each relocation takes a new
* index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
* this would mean a new BTF fd index for each entry. By pairing symbol name
* with index, we get the insn->imm, insn->off pairing that kernel uses for
* kfunc_tab, which becomes the effective limit even though all of them may
* share same index in fd_array (such that kfunc_btf_tab has 1 element).
*/
static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
{
struct ksym_desc *kdesc;
int btf_fd_idx;
kdesc = get_ksym_desc(gen, relo);
if (!kdesc)
return;
/* try to copy from existing bpf_insn */
if (kdesc->ref > 1) {
move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
kdesc->insn + offsetof(struct bpf_insn, imm));
move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
kdesc->insn + offsetof(struct bpf_insn, off));
goto log;
}
/* remember insn offset, so we can copy BTF ID and FD later */
kdesc->insn = insn;
emit_bpf_find_by_name_kind(gen, relo);
if (!relo->is_weak)
emit_check_err(gen);
/* get index in fd_array to store BTF FD at */
btf_fd_idx = add_kfunc_btf_fd(gen);
if (btf_fd_idx > INT16_MAX) {
pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
btf_fd_idx, relo->name);
gen->error = -E2BIG;
return;
}
kdesc->off = btf_fd_idx;
/* set a default value for imm */
emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
/* skip success case store if ret < 0 */
emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 1));
/* store btf_id into insn[insn_idx].imm */
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
/* load fd_array slot pointer */
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
/* skip store of BTF fd if ret < 0 */
emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 3));
/* store BTF fd in slot */
emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
/* set a default value for off */
emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
/* skip insn->off store if ret < 0 */
emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, 2));
/* skip if vmlinux BTF */
emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_9, 0, 1));
/* store index into insn[insn_idx].off */
emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
log:
if (!gen->log_level)
return;
emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
offsetof(struct bpf_insn, imm)));
emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
offsetof(struct bpf_insn, off)));
debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
relo->name, kdesc->ref);
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
relo->name, kdesc->ref);
}
/* Expects:
* BPF_REG_8 - pointer to instruction
*/
static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
{
struct ksym_desc *kdesc;
kdesc = get_ksym_desc(gen, relo);
if (!kdesc)
return;
/* try to copy from existing ldimm64 insn */
if (kdesc->ref > 1) {
move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
kdesc->insn + offsetof(struct bpf_insn, imm));
move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
goto log;
}
/* remember insn offset, so we can copy BTF ID and FD later */
kdesc->insn = insn;
emit_bpf_find_by_name_kind(gen, relo);
emit_check_err(gen);
/* store btf_id into insn[insn_idx].imm */
insn = insns + sizeof(struct bpf_insn) * relo->insn_idx +
offsetof(struct bpf_insn, imm);
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, insn));
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, 0));
if (relo->kind == BTF_KIND_VAR) {
/* store btf_obj_fd into insn[insn_idx + 1].imm */
emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
sizeof(struct bpf_insn)));
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
/* store btf_obj_fd into insn[insn_idx + 1].imm */
emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
log:
if (!gen->log_level)
return;
emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
offsetof(struct bpf_insn, imm)));
emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
offsetof(struct bpf_insn, imm)));
debug_regs(gen, BPF_REG_7, BPF_REG_9, " var (%s:count=%d): imm: %%d, fd: %%d",
relo->name, kdesc->ref);
}
static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
{
int insn;
pr_debug("gen: emit_relo (%d): %s at %d\n", relo->kind, relo->name, relo->insn_idx);
insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
switch (relo->kind) {
case BTF_KIND_VAR:
emit_relo_ksym_btf(gen, relo, insn);
break;
case BTF_KIND_FUNC:
emit_relo_kfunc_btf(gen, relo, insn);
break;
default:
pr_warn("Unknown relocation kind '%d'\n", relo->kind);
gen->error = -EDOM;
return;
}
}
@@ -571,14 +772,22 @@ static void cleanup_relos(struct bpf_gen *gen, int insns)
{
int i, insn;
for (i = 0; i < gen->relo_cnt; i++) {
if (gen->relos[i].kind != BTF_KIND_VAR)
continue;
/* close fd recorded in insn[insn_idx + 1].imm */
insn = insns +
sizeof(struct bpf_insn) * (gen->relos[i].insn_idx + 1) +
offsetof(struct bpf_insn, imm);
emit_sys_close_blob(gen, insn);
for (i = 0; i < gen->nr_ksyms; i++) {
if (gen->ksyms[i].kind == BTF_KIND_VAR) {
/* close fd recorded in insn[insn_idx + 1].imm */
insn = gen->ksyms[i].insn;
insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
emit_sys_close_blob(gen, insn);
} else { /* BTF_KIND_FUNC */
emit_sys_close_blob(gen, blob_fd_array_off(gen, gen->ksyms[i].off));
if (gen->ksyms[i].off < MAX_FD_ARRAY_SZ)
gen->nr_fd_array--;
}
}
if (gen->nr_ksyms) {
free(gen->ksyms);
gen->nr_ksyms = 0;
gen->ksyms = NULL;
}
if (gen->relo_cnt) {
free(gen->relos);
@@ -637,9 +846,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen,
/* populate union bpf_attr with a pointer to line_info */
emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
/* populate union bpf_attr fd_array with a pointer to stack where map_fds are saved */
emit_rel_store_sp(gen, attr_field(prog_load_attr, fd_array),
stack_off(map_fd[0]));
/* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
/* populate union bpf_attr with user provided log details */
move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
@@ -706,8 +914,8 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
map_update_attr = add_data(gen, &attr, attr_size);
move_stack2blob(gen, attr_field(map_update_attr, map_fd), 4,
stack_off(map_fd[map_idx]));
move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
blob_fd_array_off(gen, map_idx));
emit_rel_store(gen, attr_field(map_update_attr, key), key);
emit_rel_store(gen, attr_field(map_update_attr, value), value);
/* emit MAP_UPDATE_ELEM command */
@@ -725,8 +933,8 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
memset(&attr, 0, attr_size);
pr_debug("gen: map_freeze: idx %d\n", map_idx);
map_freeze_attr = add_data(gen, &attr, attr_size);
move_stack2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
stack_off(map_fd[map_idx]));
move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
blob_fd_array_off(gen, map_idx));
/* emit MAP_FREEZE command */
emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
debug_ret(gen, "map_freeze");

162
tools/lib/bpf/libbpf.c
View File

@@ -443,6 +443,11 @@ struct extern_desc {
/* local btf_id of the ksym extern's type. */
__u32 type_id;
/* BTF fd index to be patched in for insn->off, this is
* 0 for vmlinux BTF, index in obj->fd_array for module
* BTF
*/
__s16 btf_fd_idx;
} ksym;
};
};
@@ -454,6 +459,7 @@ struct module_btf {
char *name;
__u32 id;
int fd;
int fd_array_idx;
};
struct bpf_object {
@@ -539,6 +545,10 @@ struct bpf_object {
void *priv;
bpf_object_clear_priv_t clear_priv;
int *fd_array;
size_t fd_array_cap;
size_t fd_array_cnt;
char path[];
};
#define obj_elf_valid(o) ((o)->efile.elf)
@@ -3429,11 +3439,6 @@ static int bpf_object__collect_externs(struct bpf_object *obj)
return -ENOTSUP;
}
} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
if (btf_is_func(t) && ext->is_weak) {
pr_warn("extern weak function %s is unsupported\n",
ext->name);
return -ENOTSUP;
}
ksym_sec = sec;
ext->type = EXT_KSYM;
skip_mods_and_typedefs(obj->btf, t->type,
@@ -5406,7 +5411,13 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
case RELO_EXTERN_FUNC:
ext = &obj->externs[relo->sym_off];
insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
insn[0].imm = ext->ksym.kernel_btf_id;
if (ext->is_set) {
insn[0].imm = ext->ksym.kernel_btf_id;
insn[0].off = ext->ksym.btf_fd_idx;
} else { /* unresolved weak kfunc */
insn[0].imm = 0;
insn[0].off = 0;
}
break;
case RELO_SUBPROG_ADDR:
if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
@@ -6236,6 +6247,7 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
}
load_attr.log_level = prog->log_level;
load_attr.prog_flags = prog->prog_flags;
load_attr.fd_array = prog->obj->fd_array;
/* adjust load_attr if sec_def provides custom preload callback */
if (prog->sec_def && prog->sec_def->preload_fn) {
@@ -6348,12 +6360,12 @@ static int bpf_program__record_externs(struct bpf_program *prog)
ext->name);
return -ENOTSUP;
}
bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR,
relo->insn_idx);
bpf_gen__record_extern(obj->gen_loader, ext->name, ext->is_weak,
BTF_KIND_VAR, relo->insn_idx);
break;
case RELO_EXTERN_FUNC:
bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC,
relo->insn_idx);
bpf_gen__record_extern(obj->gen_loader, ext->name, ext->is_weak,
BTF_KIND_FUNC, relo->insn_idx);
break;
default:
continue;
@@ -6660,7 +6672,7 @@ bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, &opts));
}
int bpf_object__unload(struct bpf_object *obj)
static int bpf_object_unload(struct bpf_object *obj)
{
size_t i;
@@ -6679,6 +6691,8 @@ int bpf_object__unload(struct bpf_object *obj)
return 0;
}
int bpf_object__unload(struct bpf_object *obj) __attribute__((alias("bpf_object_unload")));
static int bpf_object__sanitize_maps(struct bpf_object *obj)
{
struct bpf_map *m;
@@ -6752,13 +6766,14 @@ out:
static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
__u16 kind, struct btf **res_btf,
int *res_btf_fd)
struct module_btf **res_mod_btf)
{
int i, id, btf_fd, err;
struct module_btf *mod_btf;
struct btf *btf;
int i, id, err;
btf = obj->btf_vmlinux;
btf_fd = 0;
mod_btf = NULL;
id = btf__find_by_name_kind(btf, ksym_name, kind);
if (id == -ENOENT) {
@@ -6767,10 +6782,10 @@ static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
return err;
for (i = 0; i < obj->btf_module_cnt; i++) {
btf = obj->btf_modules[i].btf;
/* we assume module BTF FD is always >0 */
btf_fd = obj->btf_modules[i].fd;
id = btf__find_by_name_kind(btf, ksym_name, kind);
/* we assume module_btf's BTF FD is always >0 */
mod_btf = &obj->btf_modules[i];
btf = mod_btf->btf;
id = btf__find_by_name_kind_own(btf, ksym_name, kind);
if (id != -ENOENT)
break;
}
@@ -6779,7 +6794,7 @@ static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
return -ESRCH;
*res_btf = btf;
*res_btf_fd = btf_fd;
*res_mod_btf = mod_btf;
return id;
}
@@ -6788,14 +6803,15 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
{
const struct btf_type *targ_var, *targ_type;
__u32 targ_type_id, local_type_id;
struct module_btf *mod_btf = NULL;
const char *targ_var_name;
int id, btf_fd = 0, err;
struct btf *btf = NULL;
int id, err;
id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd);
if (id == -ESRCH && ext->is_weak) {
return 0;
} else if (id < 0) {
id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
if (id < 0) {
if (id == -ESRCH && ext->is_weak)
return 0;
pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
ext->name);
return id;
@@ -6827,7 +6843,7 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
}
ext->is_set = true;
ext->ksym.kernel_btf_obj_fd = btf_fd;
ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;
ext->ksym.kernel_btf_id = id;
pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
ext->name, id, btf_kind_str(targ_var), targ_var_name);
@@ -6839,26 +6855,22 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
struct extern_desc *ext)
{
int local_func_proto_id, kfunc_proto_id, kfunc_id;
struct module_btf *mod_btf = NULL;
const struct btf_type *kern_func;
struct btf *kern_btf = NULL;
int ret, kern_btf_fd = 0;
int ret;
local_func_proto_id = ext->ksym.type_id;
kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC,
&kern_btf, &kern_btf_fd);
kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC, &kern_btf, &mod_btf);
if (kfunc_id < 0) {
pr_warn("extern (func ksym) '%s': not found in kernel BTF\n",
if (kfunc_id == -ESRCH && ext->is_weak)
return 0;
pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n",
ext->name);
return kfunc_id;
}
if (kern_btf != obj->btf_vmlinux) {
pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n",
ext->name);
return -ENOTSUP;
}
kern_func = btf__type_by_id(kern_btf, kfunc_id);
kfunc_proto_id = kern_func->type;
@@ -6870,9 +6882,30 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
return -EINVAL;
}
/* set index for module BTF fd in fd_array, if unset */
if (mod_btf && !mod_btf->fd_array_idx) {
/* insn->off is s16 */
if (obj->fd_array_cnt == INT16_MAX) {
pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n",
ext->name, mod_btf->fd_array_idx);
return -E2BIG;
}
/* Cannot use index 0 for module BTF fd */
if (!obj->fd_array_cnt)
obj->fd_array_cnt = 1;
ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
obj->fd_array_cnt + 1);
if (ret)
return ret;
mod_btf->fd_array_idx = obj->fd_array_cnt;
/* we assume module BTF FD is always >0 */
obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
}
ext->is_set = true;
ext->ksym.kernel_btf_obj_fd = kern_btf_fd;
ext->ksym.kernel_btf_id = kfunc_id;
ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;
pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
ext->name, kfunc_id);
@@ -7032,6 +7065,9 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
err = bpf_gen__finish(obj->gen_loader);
}
/* clean up fd_array */
zfree(&obj->fd_array);
/* clean up module BTFs */
for (i = 0; i < obj->btf_module_cnt; i++) {
close(obj->btf_modules[i].fd);
@@ -7056,7 +7092,7 @@ out:
if (obj->maps[i].pinned && !obj->maps[i].reused)
bpf_map__unpin(&obj->maps[i], NULL);
bpf_object__unload(obj);
bpf_object_unload(obj);
pr_warn("failed to load object '%s'\n", obj->path);
return libbpf_err(err);
}
@@ -7665,7 +7701,7 @@ void bpf_object__close(struct bpf_object *obj)
bpf_gen__free(obj->gen_loader);
bpf_object__elf_finish(obj);
bpf_object__unload(obj);
bpf_object_unload(obj);
btf__free(obj->btf);
btf_ext__free(obj->btf_ext);
@@ -7799,6 +7835,12 @@ __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
struct bpf_program *
bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
{
return bpf_object__next_program(obj, prev);
}
struct bpf_program *
bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
{
struct bpf_program *prog = prev;
@@ -7811,6 +7853,12 @@ bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
struct bpf_program *
bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
{
return bpf_object__prev_program(obj, next);
}
struct bpf_program *
bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
{
struct bpf_program *prog = next;
@@ -8030,6 +8078,8 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("tp/", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
SEC_DEF("raw_tp/", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
SEC_DEF("raw_tracepoint.w/", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
SEC_DEF("raw_tp.w/", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp),
SEC_DEF("tp_btf/", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace),
SEC_DEF("fentry/", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace),
SEC_DEF("fmod_ret/", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace),
@@ -8743,6 +8793,12 @@ __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
struct bpf_map *
bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
{
return bpf_object__next_map(obj, prev);
}
struct bpf_map *
bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
{
if (prev == NULL)
return obj->maps;
@@ -8752,6 +8808,12 @@ bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
struct bpf_map *
bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
{
return bpf_object__prev_map(obj, next);
}
struct bpf_map *
bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
{
if (next == NULL) {
if (!obj->nr_maps)
@@ -9787,12 +9849,26 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *pr
static struct bpf_link *attach_raw_tp(const struct bpf_program *prog, long cookie)
{
const char *tp_name;
static const char *const prefixes[] = {
"raw_tp/",
"raw_tracepoint/",
"raw_tp.w/",
"raw_tracepoint.w/",
};
size_t i;
const char *tp_name = NULL;
if (str_has_pfx(prog->sec_name, "raw_tp/"))
tp_name = prog->sec_name + sizeof("raw_tp/") - 1;
else
tp_name = prog->sec_name + sizeof("raw_tracepoint/") - 1;
for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
if (str_has_pfx(prog->sec_name, prefixes[i])) {
tp_name = prog->sec_name + strlen(prefixes[i]);
break;
}
}
if (!tp_name) {
pr_warn("prog '%s': invalid section name '%s'\n",
prog->name, prog->sec_name);
return libbpf_err_ptr(-EINVAL);
}
return bpf_program__attach_raw_tracepoint(prog, tp_name);
}

36
tools/lib/bpf/libbpf.h
View File

@@ -150,6 +150,7 @@ struct bpf_object_load_attr {
/* Load/unload object into/from kernel */
LIBBPF_API int bpf_object__load(struct bpf_object *obj);
LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr);
LIBBPF_DEPRECATED_SINCE(0, 6, "bpf_object__unload() is deprecated, use bpf_object__close() instead")
LIBBPF_API int bpf_object__unload(struct bpf_object *obj);
LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj);
@@ -189,16 +190,22 @@ LIBBPF_API int libbpf_find_vmlinux_btf_id(const char *name,
/* Accessors of bpf_program */
struct bpf_program;
LIBBPF_API struct bpf_program *bpf_program__next(struct bpf_program *prog,
const struct bpf_object *obj);
LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_program() instead")
struct bpf_program *bpf_program__next(struct bpf_program *prog,
const struct bpf_object *obj);
LIBBPF_API struct bpf_program *
bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prog);
#define bpf_object__for_each_program(pos, obj) \
for ((pos) = bpf_program__next(NULL, (obj)); \
(pos) != NULL; \
(pos) = bpf_program__next((pos), (obj)))
#define bpf_object__for_each_program(pos, obj) \
for ((pos) = bpf_object__next_program((obj), NULL); \
(pos) != NULL; \
(pos) = bpf_object__next_program((obj), (pos)))
LIBBPF_API struct bpf_program *bpf_program__prev(struct bpf_program *prog,
const struct bpf_object *obj);
LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__prev_program() instead")
struct bpf_program *bpf_program__prev(struct bpf_program *prog,
const struct bpf_object *obj);
LIBBPF_API struct bpf_program *
bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *prog);
typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *);
@@ -502,16 +509,21 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name);
LIBBPF_API struct bpf_map *
bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset);
LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_map() instead")
struct bpf_map *bpf_map__next(const struct bpf_map *map, const struct bpf_object *obj);
LIBBPF_API struct bpf_map *
bpf_map__next(const struct bpf_map *map, const struct bpf_object *obj);
bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *map);
#define bpf_object__for_each_map(pos, obj) \
for ((pos) = bpf_map__next(NULL, (obj)); \
for ((pos) = bpf_object__next_map((obj), NULL); \
(pos) != NULL; \
(pos) = bpf_map__next((pos), (obj)))
(pos) = bpf_object__next_map((obj), (pos)))
#define bpf_map__for_each bpf_object__for_each_map
LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__prev_map() instead")
struct bpf_map *bpf_map__prev(const struct bpf_map *map, const struct bpf_object *obj);
LIBBPF_API struct bpf_map *
bpf_map__prev(const struct bpf_map *map, const struct bpf_object *obj);
bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map);
/**
* @brief **bpf_map__fd()** gets the file descriptor of the passed

5
tools/lib/bpf/libbpf.map
View File

@@ -389,5 +389,10 @@ LIBBPF_0.5.0 {
LIBBPF_0.6.0 {
global:
bpf_object__next_map;
bpf_object__next_program;
bpf_object__prev_map;
bpf_object__prev_program;
btf__add_btf;
btf__add_tag;
} LIBBPF_0.5.0;

3
tools/lib/bpf/libbpf_internal.h
View File

@@ -298,6 +298,7 @@ struct bpf_prog_load_params {
__u32 log_level;
char *log_buf;
size_t log_buf_sz;
int *fd_array;
};
int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr);
@@ -408,6 +409,8 @@ int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ct
int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx);
int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx);
int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx);
__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
__u32 kind);
extern enum libbpf_strict_mode libbpf_mode;

35
tools/testing/selftests/bpf/Makefile
View File

@@ -122,12 +122,15 @@ BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a
ifneq ($(CROSS_COMPILE),)
HOST_BUILD_DIR := $(BUILD_DIR)/host
HOST_SCRATCH_DIR := $(OUTPUT)/host-tools
HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include
else
HOST_BUILD_DIR := $(BUILD_DIR)
HOST_SCRATCH_DIR := $(SCRATCH_DIR)
HOST_INCLUDE_DIR := $(INCLUDE_DIR)
endif
HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a
RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids
RUNQSLOWER_OUTPUT := $(BUILD_DIR)/runqslower/
VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \
$(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \
@@ -152,7 +155,7 @@ $(notdir $(TEST_GEN_PROGS) \
# sort removes libbpf duplicates when not cross-building
MAKE_DIRS := $(sort $(BUILD_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \
$(HOST_BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/resolve_btfids \
$(INCLUDE_DIR))
$(RUNQSLOWER_OUTPUT) $(INCLUDE_DIR))
$(MAKE_DIRS):
$(call msg,MKDIR,,$@)
$(Q)mkdir -p $@
@@ -181,11 +184,13 @@ $(OUTPUT)/test_stub.o: test_stub.c $(BPFOBJ)
DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool
$(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL)
$(Q)$(MAKE) $(submake_extras) -C $(TOOLSDIR)/bpf/runqslower \
OUTPUT=$(SCRATCH_DIR)/ VMLINUX_BTF=$(VMLINUX_BTF) \
BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) && \
cp $(SCRATCH_DIR)/runqslower $@
$(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C $(TOOLSDIR)/bpf/runqslower \
OUTPUT=$(RUNQSLOWER_OUTPUT) VMLINUX_BTF=$(VMLINUX_BTF) \
BPFTOOL_OUTPUT=$(BUILD_DIR)/bpftool/ \
BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf \
BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) && \
cp $(RUNQSLOWER_OUTPUT)runqslower $@
TEST_GEN_PROGS_EXTENDED += $(DEFAULT_BPFTOOL)
@@ -209,7 +214,9 @@ $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
CC=$(HOSTCC) LD=$(HOSTLD) \
EXTRA_CFLAGS='-g -O0' \
OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install
LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/ \
prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install-bin
all: docs
@@ -225,7 +232,7 @@ docs-clean:
$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
../../../include/uapi/linux/bpf.h \
| $(INCLUDE_DIR) $(BUILD_DIR)/libbpf
| $(BUILD_DIR)/libbpf
$(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \
EXTRA_CFLAGS='-g -O0' \
DESTDIR=$(SCRATCH_DIR) prefix= all install_headers
@@ -233,7 +240,7 @@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
ifneq ($(BPFOBJ),$(HOST_BPFOBJ))
$(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
../../../include/uapi/linux/bpf.h \
| $(INCLUDE_DIR) $(HOST_BUILD_DIR)/libbpf
| $(HOST_BUILD_DIR)/libbpf
$(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \
EXTRA_CFLAGS='-g -O0' \
OUTPUT=$(HOST_BUILD_DIR)/libbpf/ CC=$(HOSTCC) LD=$(HOSTLD) \
@@ -258,6 +265,7 @@ $(RESOLVE_BTFIDS): $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/resolve_btfids \
$(TOOLSDIR)/lib/str_error_r.c
$(Q)$(MAKE) $(submake_extras) -C $(TOOLSDIR)/bpf/resolve_btfids \
CC=$(HOSTCC) LD=$(HOSTLD) AR=$(HOSTAR) \
LIBBPF_INCLUDE=$(HOST_INCLUDE_DIR) \
OUTPUT=$(HOST_BUILD_DIR)/resolve_btfids/ BPFOBJ=$(HOST_BPFOBJ)
# Get Clang's default includes on this system, as opposed to those seen by
@@ -315,8 +323,9 @@ LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \
linked_vars.skel.h linked_maps.skel.h
LSKELS := kfunc_call_test.c fentry_test.c fexit_test.c fexit_sleep.c \
test_ksyms_module.c test_ringbuf.c atomics.c trace_printk.c \
trace_vprintk.c
test_ringbuf.c atomics.c trace_printk.c trace_vprintk.c
# Generate both light skeleton and libbpf skeleton for these
LSKELS_EXTRA := test_ksyms_module.c
SKEL_BLACKLIST += $$(LSKELS)
test_static_linked.skel.h-deps := test_static_linked1.o test_static_linked2.o
@@ -346,7 +355,7 @@ TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, $$(TRUNNER_BPF_SRCS)
TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h, \
$$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\
$$(TRUNNER_BPF_SRCS)))
TRUNNER_BPF_LSKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS))
TRUNNER_BPF_LSKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.lskel.h, $$(LSKELS) $$(LSKELS_EXTRA))
TRUNNER_BPF_SKELS_LINKED := $$(addprefix $$(TRUNNER_OUTPUT)/,$(LINKED_SKELS))
TEST_GEN_FILES += $$(TRUNNER_BPF_OBJS)
@@ -454,7 +463,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \
| $(TRUNNER_BINARY)-extras
$$(call msg,BINARY,,$$@)
$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) -o $$@
$(Q)$(RESOLVE_BTFIDS) --no-fail --btf $(TRUNNER_OUTPUT)/btf_data.o $$@
$(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.o $$@
endef

15
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h
View File

@@ -34,6 +34,21 @@ DECLARE_TRACE(bpf_testmod_test_write_bare,
TP_ARGS(task, ctx)
);
#undef BPF_TESTMOD_DECLARE_TRACE
#ifdef DECLARE_TRACE_WRITABLE
#define BPF_TESTMOD_DECLARE_TRACE(call, proto, args, size) \
DECLARE_TRACE_WRITABLE(call, PARAMS(proto), PARAMS(args), size)
#else
#define BPF_TESTMOD_DECLARE_TRACE(call, proto, args, size) \
DECLARE_TRACE(call, PARAMS(proto), PARAMS(args))
#endif
BPF_TESTMOD_DECLARE_TRACE(bpf_testmod_test_writable_bare,
TP_PROTO(struct bpf_testmod_test_writable_ctx *ctx),
TP_ARGS(ctx),
sizeof(struct bpf_testmod_test_writable_ctx)
);
#endif /* _BPF_TESTMOD_EVENTS_H */
#undef TRACE_INCLUDE_PATH

33
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
View File

@@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/error-injection.h>
#include <linux/init.h>
#include <linux/module.h>
@@ -13,6 +15,12 @@
DEFINE_PER_CPU(int, bpf_testmod_ksym_percpu) = 123;
noinline void
bpf_testmod_test_mod_kfunc(int i)
{
*(int *)this_cpu_ptr(&bpf_testmod_ksym_percpu) = i;
}
noinline int bpf_testmod_loop_test(int n)
{
int i, sum = 0;
@@ -42,6 +50,16 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj,
if (bpf_testmod_loop_test(101) > 100)
trace_bpf_testmod_test_read(current, &ctx);
/* Magic number to enable writable tp */
if (len == 64) {
struct bpf_testmod_test_writable_ctx writable = {
.val = 1024,
};
trace_bpf_testmod_test_writable_bare(&writable);
if (writable.early_ret)
return snprintf(buf, len, "%d\n", writable.val);
}
return -EIO; /* always fail */
}
EXPORT_SYMBOL(bpf_testmod_test_read);
@@ -71,13 +89,26 @@ static struct bin_attribute bin_attr_bpf_testmod_file __ro_after_init = {
.write = bpf_testmod_test_write,
};
BTF_SET_START(bpf_testmod_kfunc_ids)
BTF_ID(func, bpf_testmod_test_mod_kfunc)
BTF_SET_END(bpf_testmod_kfunc_ids)
static DEFINE_KFUNC_BTF_ID_SET(&bpf_testmod_kfunc_ids, bpf_testmod_kfunc_btf_set);
static int bpf_testmod_init(void)
{
return sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
int ret;
ret = sysfs_create_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
if (ret)
return ret;
register_kfunc_btf_id_set(&prog_test_kfunc_list, &bpf_testmod_kfunc_btf_set);
return 0;
}
static void bpf_testmod_exit(void)
{
unregister_kfunc_btf_id_set(&prog_test_kfunc_list, &bpf_testmod_kfunc_btf_set);
return sysfs_remove_bin_file(kernel_kobj, &bin_attr_bpf_testmod_file);
}

5
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h
View File

@@ -17,4 +17,9 @@ struct bpf_testmod_test_write_ctx {
size_t len;
};
struct bpf_testmod_test_writable_ctx {
bool early_ret;
int val;
};
#endif /* _BPF_TESTMOD_H */

6
tools/testing/selftests/bpf/cgroup_helpers.c
View File

@@ -11,6 +11,7 @@
#include <fcntl.h>
#include <unistd.h>
#include <ftw.h>
#include <unistd.h>
#include "cgroup_helpers.h"
@@ -33,10 +34,9 @@
#define CGROUP_MOUNT_DFLT "/sys/fs/cgroup"
#define NETCLS_MOUNT_PATH CGROUP_MOUNT_DFLT "/net_cls"
#define CGROUP_WORK_DIR "/cgroup-test-work-dir"
#define format_cgroup_path(buf, path) \
snprintf(buf, sizeof(buf), "%s%s%s", CGROUP_MOUNT_PATH, \
CGROUP_WORK_DIR, path)
snprintf(buf, sizeof(buf), "%s%s%d%s", CGROUP_MOUNT_PATH, \
CGROUP_WORK_DIR, getpid(), path)
#define format_classid_path(buf) \
snprintf(buf, sizeof(buf), "%s%s", NETCLS_MOUNT_PATH, \

2
tools/testing/selftests/bpf/cgroup_helpers.h
View File

@@ -26,4 +26,4 @@ int join_classid(void);
int setup_classid_environment(void);
void cleanup_classid_environment(void);
#endif /* __CGROUP_HELPERS_H */
#endif /* __CGROUP_HELPERS_H */

1
tools/testing/selftests/bpf/prog_tests/atomics.c
View File

@@ -225,6 +225,7 @@ void test_atomics(void)
test__skip();
goto cleanup;
}
skel->bss->pid = getpid();
if (test__start_subtest("add"))
test_add(skel);

2
tools/testing/selftests/bpf/prog_tests/bpf_iter_setsockopt.c
View File

@@ -179,7 +179,7 @@ done:
free_fds(est_fds, nr_est);
}
void test_bpf_iter_setsockopt(void)
void serial_test_bpf_iter_setsockopt(void)
{
struct bpf_iter_setsockopt *iter_skel = NULL;
struct bpf_cubic *cubic_skel = NULL;

2
tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
View File

@@ -3,7 +3,7 @@
#define nr_iters 2
void test_bpf_obj_id(void)
void serial_test_bpf_obj_id(void)
{
const __u64 array_magic_value = 0xfaceb00c;
const __u32 array_key = 0;

2
tools/testing/selftests/bpf/prog_tests/btf.c
View File

@@ -4511,7 +4511,7 @@ static void do_test_file(unsigned int test_num)
if (CHECK(err, "obj: %d", err))
return;
prog = bpf_program__next(NULL, obj);
prog = bpf_object__next_program(obj, NULL);
if (CHECK(!prog, "Cannot find bpf_prog")) {
err = -1;
goto done;

141
tools/testing/selftests/bpf/prog_tests/btf_write.c
View File

@@ -4,19 +4,15 @@
#include <bpf/btf.h>
#include "btf_helpers.h"
void test_btf_write() {
static void gen_btf(struct btf *btf)
{
const struct btf_var_secinfo *vi;
const struct btf_type *t;
const struct btf_member *m;
const struct btf_enum *v;
const struct btf_param *p;
struct btf *btf;
int id, err, str_off;
btf = btf__new_empty();
if (!ASSERT_OK_PTR(btf, "new_empty"))
return;
str_off = btf__find_str(btf, "int");
ASSERT_EQ(str_off, -ENOENT, "int_str_missing_off");
@@ -301,6 +297,139 @@ void test_btf_write() {
ASSERT_EQ(btf_tag(t)->component_idx, 1, "tag_component_idx");
ASSERT_STREQ(btf_type_raw_dump(btf, 19),
"[19] TAG 'tag2' type_id=14 component_idx=1", "raw_dump");
}
static void test_btf_add()
{
struct btf *btf;
btf = btf__new_empty();
if (!ASSERT_OK_PTR(btf, "new_empty"))
return;
gen_btf(btf);
VALIDATE_RAW_BTF(
btf,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=1",
"[3] CONST '(anon)' type_id=5",
"[4] VOLATILE '(anon)' type_id=3",
"[5] RESTRICT '(anon)' type_id=4",
"[6] ARRAY '(anon)' type_id=2 index_type_id=1 nr_elems=10",
"[7] STRUCT 's1' size=8 vlen=2\n"
"\t'f1' type_id=1 bits_offset=0\n"
"\t'f2' type_id=1 bits_offset=32 bitfield_size=16",
"[8] UNION 'u1' size=8 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0 bitfield_size=16",
"[9] ENUM 'e1' size=4 vlen=2\n"
"\t'v1' val=1\n"
"\t'v2' val=2",
"[10] FWD 'struct_fwd' fwd_kind=struct",
"[11] FWD 'union_fwd' fwd_kind=union",
"[12] ENUM 'enum_fwd' size=4 vlen=0",
"[13] TYPEDEF 'typedef1' type_id=1",
"[14] FUNC 'func1' type_id=15 linkage=global",
"[15] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n"
"\t'p1' type_id=1\n"
"\t'p2' type_id=2",
"[16] VAR 'var1' type_id=1, linkage=global-alloc",
"[17] DATASEC 'datasec1' size=12 vlen=1\n"
"\ttype_id=1 offset=4 size=8",
"[18] TAG 'tag1' type_id=16 component_idx=-1",
"[19] TAG 'tag2' type_id=14 component_idx=1");
btf__free(btf);
}
static void test_btf_add_btf()
{
struct btf *btf1 = NULL, *btf2 = NULL;
int id;
btf1 = btf__new_empty();
if (!ASSERT_OK_PTR(btf1, "btf1"))
return;
btf2 = btf__new_empty();
if (!ASSERT_OK_PTR(btf2, "btf2"))
goto cleanup;
gen_btf(btf1);
gen_btf(btf2);
id = btf__add_btf(btf1, btf2);
if (!ASSERT_EQ(id, 20, "id"))
goto cleanup;
VALIDATE_RAW_BTF(
btf1,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=1",
"[3] CONST '(anon)' type_id=5",
"[4] VOLATILE '(anon)' type_id=3",
"[5] RESTRICT '(anon)' type_id=4",
"[6] ARRAY '(anon)' type_id=2 index_type_id=1 nr_elems=10",
"[7] STRUCT 's1' size=8 vlen=2\n"
"\t'f1' type_id=1 bits_offset=0\n"
"\t'f2' type_id=1 bits_offset=32 bitfield_size=16",
"[8] UNION 'u1' size=8 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0 bitfield_size=16",
"[9] ENUM 'e1' size=4 vlen=2\n"
"\t'v1' val=1\n"
"\t'v2' val=2",
"[10] FWD 'struct_fwd' fwd_kind=struct",
"[11] FWD 'union_fwd' fwd_kind=union",
"[12] ENUM 'enum_fwd' size=4 vlen=0",
"[13] TYPEDEF 'typedef1' type_id=1",
"[14] FUNC 'func1' type_id=15 linkage=global",
"[15] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n"
"\t'p1' type_id=1\n"
"\t'p2' type_id=2",
"[16] VAR 'var1' type_id=1, linkage=global-alloc",
"[17] DATASEC 'datasec1' size=12 vlen=1\n"
"\ttype_id=1 offset=4 size=8",
"[18] TAG 'tag1' type_id=16 component_idx=-1",
"[19] TAG 'tag2' type_id=14 component_idx=1",
/* types appended from the second BTF */
"[20] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[21] PTR '(anon)' type_id=20",
"[22] CONST '(anon)' type_id=24",
"[23] VOLATILE '(anon)' type_id=22",
"[24] RESTRICT '(anon)' type_id=23",
"[25] ARRAY '(anon)' type_id=21 index_type_id=20 nr_elems=10",
"[26] STRUCT 's1' size=8 vlen=2\n"
"\t'f1' type_id=20 bits_offset=0\n"
"\t'f2' type_id=20 bits_offset=32 bitfield_size=16",
"[27] UNION 'u1' size=8 vlen=1\n"
"\t'f1' type_id=20 bits_offset=0 bitfield_size=16",
"[28] ENUM 'e1' size=4 vlen=2\n"
"\t'v1' val=1\n"
"\t'v2' val=2",
"[29] FWD 'struct_fwd' fwd_kind=struct",
"[30] FWD 'union_fwd' fwd_kind=union",
"[31] ENUM 'enum_fwd' size=4 vlen=0",
"[32] TYPEDEF 'typedef1' type_id=20",
"[33] FUNC 'func1' type_id=34 linkage=global",
"[34] FUNC_PROTO '(anon)' ret_type_id=20 vlen=2\n"
"\t'p1' type_id=20\n"
"\t'p2' type_id=21",
"[35] VAR 'var1' type_id=20, linkage=global-alloc",
"[36] DATASEC 'datasec1' size=12 vlen=1\n"
"\ttype_id=20 offset=4 size=8",
"[37] TAG 'tag1' type_id=35 component_idx=-1",
"[38] TAG 'tag2' type_id=33 component_idx=1");
cleanup:
btf__free(btf1);
btf__free(btf2);
}
void test_btf_write()
{
if (test__start_subtest("btf_add"))
test_btf_add();
if (test__start_subtest("btf_add_btf"))
test_btf_add_btf();
}

2
tools/testing/selftests/bpf/prog_tests/cg_storage_multi.c
View File

@@ -363,7 +363,7 @@ close_bpf_object:
cg_storage_multi_shared__destroy(obj);
}
void test_cg_storage_multi(void)
void serial_test_cg_storage_multi(void)
{
int parent_cgroup_fd = -1, child_cgroup_fd = -1;

2
tools/testing/selftests/bpf/prog_tests/cgroup_attach_autodetach.c
View File

@@ -21,7 +21,7 @@ static int prog_load(void)
bpf_log_buf, BPF_LOG_BUF_SIZE);
}
void test_cgroup_attach_autodetach(void)
void serial_test_cgroup_attach_autodetach(void)
{
__u32 duration = 0, prog_cnt = 4, attach_flags;
int allow_prog[2] = {-1};

2
tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c
View File

@@ -74,7 +74,7 @@ static int prog_load_cnt(int verdict, int val)
return ret;
}
void test_cgroup_attach_multi(void)
void serial_test_cgroup_attach_multi(void)
{
__u32 prog_ids[4], prog_cnt = 0, attach_flags, saved_prog_id;
int cg1 = 0, cg2 = 0, cg3 = 0, cg4 = 0, cg5 = 0, key = 0;

2
tools/testing/selftests/bpf/prog_tests/cgroup_attach_override.c
View File

@@ -23,7 +23,7 @@ static int prog_load(int verdict)
bpf_log_buf, BPF_LOG_BUF_SIZE);
}
void test_cgroup_attach_override(void)
void serial_test_cgroup_attach_override(void)
{
int drop_prog = -1, allow_prog = -1, foo = -1, bar = -1;
__u32 duration = 0;

2
tools/testing/selftests/bpf/prog_tests/cgroup_link.c
View File

@@ -24,7 +24,7 @@ int ping_and_check(int exp_calls, int exp_alt_calls)
return 0;
}
void test_cgroup_link(void)
void serial_test_cgroup_link(void)
{
struct {
const char *path;

2
tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c
View File

@@ -46,7 +46,7 @@ void test_cgroup_v1v2(void)
{
struct network_helper_opts opts = {};
int server_fd, client_fd, cgroup_fd;
static const int port = 60123;
static const int port = 60120;
/* Step 1: Check base connectivity works without any BPF. */
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);

2
tools/testing/selftests/bpf/prog_tests/check_mtu.c
View File

@@ -195,7 +195,7 @@ cleanup:
test_check_mtu__destroy(skel);
}
void test_check_mtu(void)
void serial_test_check_mtu(void)
{
__u32 mtu_lo;

9
tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
View File

@@ -285,7 +285,7 @@ static void test_fmod_ret_freplace(void)
if (!ASSERT_OK_PTR(freplace_obj, "freplace_obj_open"))
goto out;
prog = bpf_program__next(NULL, freplace_obj);
prog = bpf_object__next_program(freplace_obj, NULL);
err = bpf_program__set_attach_target(prog, pkt_fd, NULL);
ASSERT_OK(err, "freplace__set_attach_target");
@@ -302,7 +302,7 @@ static void test_fmod_ret_freplace(void)
goto out;
attach_prog_fd = bpf_program__fd(prog);
prog = bpf_program__next(NULL, fmod_obj);
prog = bpf_object__next_program(fmod_obj, NULL);
err = bpf_program__set_attach_target(prog, attach_prog_fd, NULL);
ASSERT_OK(err, "fmod_ret_set_attach_target");
@@ -352,7 +352,7 @@ static void test_obj_load_failure_common(const char *obj_file,
if (!ASSERT_OK_PTR(obj, "obj_open"))
goto close_prog;
prog = bpf_program__next(NULL, obj);
prog = bpf_object__next_program(obj, NULL);
err = bpf_program__set_attach_target(prog, pkt_fd, NULL);
ASSERT_OK(err, "set_attach_target");
@@ -380,7 +380,8 @@ static void test_func_map_prog_compatibility(void)
"./test_attach_probe.o");
}
void test_fexit_bpf2bpf(void)
/* NOTE: affect other tests, must run in serial mode */
void serial_test_fexit_bpf2bpf(void)
{
if (test__start_subtest("target_no_callees"))
test_target_no_callees();

2
tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c
View File

@@ -2,7 +2,7 @@
#include <test_progs.h>
#include <network_helpers.h>
void test_flow_dissector_load_bytes(void)
void serial_test_flow_dissector_load_bytes(void)
{
struct bpf_flow_keys flow_keys;
__u32 duration = 0, retval, size;

2
tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c
View File

@@ -628,7 +628,7 @@ out_close:
}
}
void test_flow_dissector_reattach(void)
void serial_test_flow_dissector_reattach(void)
{
int err, new_net, saved_net;

36
tools/testing/selftests/bpf/prog_tests/get_branch_snapshot.c
View File

@@ -6,6 +6,30 @@
static int *pfd_array;
static int cpu_cnt;
static bool is_hypervisor(void)
{
char *line = NULL;
bool ret = false;
size_t len;
FILE *fp;
fp = fopen("/proc/cpuinfo", "r");
if (!fp)
return false;
while (getline(&line, &len, fp) != -1) {
if (!strncmp(line, "flags", 5)) {
if (strstr(line, "hypervisor") != NULL)
ret = true;
break;
}
}
free(line);
fclose(fp);
return ret;
}
static int create_perf_events(void)
{
struct perf_event_attr attr = {0};
@@ -49,7 +73,7 @@ static void close_perf_events(void)
free(pfd_array);
}
void test_get_branch_snapshot(void)
void serial_test_get_branch_snapshot(void)
{
struct get_branch_snapshot *skel = NULL;
int err;
@@ -83,6 +107,16 @@ void test_get_branch_snapshot(void)
goto cleanup;
}
if (is_hypervisor()) {
/* As of today, LBR in hypervisor cannot be stopped before
* too many entries are flushed. Skip the hit/waste test
* for now in hypervisor until we optimize the LBR in
* hypervisor.
*/
test__skip();
goto cleanup;
}
ASSERT_GT(skel->bss->test1_hits, 6, "find_looptest_in_lbr");
/* Given we stop LBR in software, we will waste a few entries.

3
tools/testing/selftests/bpf/prog_tests/kfree_skb.c
View File

@@ -48,7 +48,8 @@ static void on_sample(void *ctx, int cpu, void *data, __u32 size)
*(bool *)ctx = true;
}
void test_kfree_skb(void)
/* TODO: fix kernel panic caused by this test in parallel mode */
void serial_test_kfree_skb(void)
{
struct __sk_buff skb = {};
struct bpf_prog_test_run_attr tattr = {

29
tools/testing/selftests/bpf/prog_tests/ksyms_module.c
View File

@@ -2,30 +2,29 @@
/* Copyright (c) 2021 Facebook */
#include <test_progs.h>
#include <bpf/libbpf.h>
#include <bpf/btf.h>
#include <network_helpers.h>
#include "test_ksyms_module.lskel.h"
static int duration;
void test_ksyms_module(void)
{
struct test_ksyms_module* skel;
struct test_ksyms_module *skel;
int retval;
int err;
skel = test_ksyms_module__open_and_load();
if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
if (!env.has_testmod) {
test__skip();
return;
}
err = test_ksyms_module__attach(skel);
if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
skel = test_ksyms_module__open_and_load();
if (!ASSERT_OK_PTR(skel, "test_ksyms_module__open_and_load"))
return;
err = bpf_prog_test_run(skel->progs.load.prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
NULL, NULL, (__u32 *)&retval, NULL);
if (!ASSERT_OK(err, "bpf_prog_test_run"))
goto cleanup;
usleep(1);
ASSERT_EQ(skel->bss->triggered, true, "triggered");
ASSERT_EQ(skel->bss->out_mod_ksym_global, 123, "global_ksym_val");
ASSERT_EQ(retval, 0, "retval");
ASSERT_EQ(skel->bss->out_bpf_testmod_ksym, 42, "bpf_testmod_ksym");
cleanup:
test_ksyms_module__destroy(skel);
}

28
tools/testing/selftests/bpf/prog_tests/ksyms_module_libbpf.c Normal file
View File

@@ -0,0 +1,28 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include <network_helpers.h>
#include "test_ksyms_module.skel.h"
void test_ksyms_module_libbpf(void)
{
struct test_ksyms_module *skel;
int retval, err;
if (!env.has_testmod) {
test__skip();
return;
}
skel = test_ksyms_module__open_and_load();
if (!ASSERT_OK_PTR(skel, "test_ksyms_module__open"))
return;
err = bpf_prog_test_run(bpf_program__fd(skel->progs.load), 1, &pkt_v4,
sizeof(pkt_v4), NULL, NULL, (__u32 *)&retval, NULL);
if (!ASSERT_OK(err, "bpf_prog_test_run"))
goto cleanup;
ASSERT_EQ(retval, 0, "retval");
ASSERT_EQ(skel->bss->out_bpf_testmod_ksym, 42, "bpf_testmod_ksym");
cleanup:
test_ksyms_module__destroy(skel);
}

2
tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c
View File

@@ -541,7 +541,7 @@ close_servers:
}
}
void test_migrate_reuseport(void)
void serial_test_migrate_reuseport(void)
{
struct test_migrate_reuseport *skel;
int i;

3
tools/testing/selftests/bpf/prog_tests/modify_return.c
View File

@@ -53,7 +53,8 @@ cleanup:
modify_return__destroy(skel);
}
void test_modify_return(void)
/* TODO: conflict with get_func_ip_test */
void serial_test_modify_return(void)
{
run_test(0 /* input_retval */,
1 /* want_side_effect */,

35
tools/testing/selftests/bpf/prog_tests/module_attach.c
View File

@@ -2,10 +2,36 @@
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include <stdbool.h>
#include "test_module_attach.skel.h"
static int duration;
static int trigger_module_test_writable(int *val)
{
int fd, err;
char buf[65];
ssize_t rd;
fd = open(BPF_TESTMOD_TEST_FILE, O_RDONLY);
err = -errno;
if (!ASSERT_GE(fd, 0, "testmode_file_open"))
return err;
rd = read(fd, buf, sizeof(buf) - 1);
err = -errno;
if (!ASSERT_GT(rd, 0, "testmod_file_rd_val")) {
close(fd);
return err;
}
buf[rd] = '\0';
*val = strtol(buf, NULL, 0);
close(fd);
return 0;
}
static int delete_module(const char *name, int flags)
{
return syscall(__NR_delete_module, name, flags);
@@ -19,6 +45,7 @@ void test_module_attach(void)
struct test_module_attach__bss *bss;
struct bpf_link *link;
int err;
int writable_val = 0;
skel = test_module_attach__open();
if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
@@ -51,6 +78,14 @@ void test_module_attach(void)
ASSERT_EQ(bss->fexit_ret, -EIO, "fexit_tet");
ASSERT_EQ(bss->fmod_ret_read_sz, READ_SZ, "fmod_ret");
bss->raw_tp_writable_bare_early_ret = true;
bss->raw_tp_writable_bare_out_val = 0xf1f2f3f4;
ASSERT_OK(trigger_module_test_writable(&writable_val),
"trigger_writable");
ASSERT_EQ(bss->raw_tp_writable_bare_in_val, 1024, "writable_test_in");
ASSERT_EQ(bss->raw_tp_writable_bare_out_val, writable_val,
"writable_test_out");
test_module_attach__detach(skel);
/* attach fentry/fexit and make sure it get's module reference */

3
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c
View File

@@ -78,7 +78,8 @@ static void test_ns_current_pid_tgid_new_ns(void)
return;
}
void test_ns_current_pid_tgid(void)
/* TODO: use a different tracepoint */
void serial_test_ns_current_pid_tgid(void)
{
if (test__start_subtest("ns_current_pid_tgid_root_ns"))
test_current_pid_tgid(NULL);

2
tools/testing/selftests/bpf/prog_tests/perf_buffer.c
View File

@@ -43,7 +43,7 @@ int trigger_on_cpu(int cpu)
return 0;
}
void test_perf_buffer(void)
void serial_test_perf_buffer(void)
{
int err, on_len, nr_on_cpus = 0, nr_cpus, i;
struct perf_buffer_opts pb_opts = {};

3
tools/testing/selftests/bpf/prog_tests/perf_link.c
View File

@@ -23,7 +23,8 @@ static void burn_cpu(void)
++j;
}
void test_perf_link(void)
/* TODO: often fails in concurrent mode */
void serial_test_perf_link(void)
{
struct test_perf_link *skel = NULL;
struct perf_event_attr attr;

3
tools/testing/selftests/bpf/prog_tests/probe_user.c
View File

@@ -1,7 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
void test_probe_user(void)
/* TODO: corrupts other tests uses connect() */
void serial_test_probe_user(void)
{
const char *prog_name = "handle_sys_connect";
const char *obj_file = "./test_probe_user.o";

3
tools/testing/selftests/bpf/prog_tests/raw_tp_writable_test_run.c
View File

@@ -3,7 +3,8 @@
#include <test_progs.h>
#include <linux/nbd.h>
void test_raw_tp_writable_test_run(void)
/* NOTE: conflict with other tests. */
void serial_test_raw_tp_writable_test_run(void)
{
__u32 duration = 0;
char error[4096];

10
tools/testing/selftests/bpf/prog_tests/recursion.c
View File

@@ -20,18 +20,18 @@ void test_recursion(void)
goto out;
ASSERT_EQ(skel->bss->pass1, 0, "pass1 == 0");
bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash1), &key, 0);
bpf_map_delete_elem(bpf_map__fd(skel->maps.hash1), &key);
ASSERT_EQ(skel->bss->pass1, 1, "pass1 == 1");
bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash1), &key, 0);
bpf_map_delete_elem(bpf_map__fd(skel->maps.hash1), &key);
ASSERT_EQ(skel->bss->pass1, 2, "pass1 == 2");
ASSERT_EQ(skel->bss->pass2, 0, "pass2 == 0");
bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash2), &key, 0);
bpf_map_delete_elem(bpf_map__fd(skel->maps.hash2), &key);
ASSERT_EQ(skel->bss->pass2, 1, "pass2 == 1");
bpf_map_lookup_elem(bpf_map__fd(skel->maps.hash2), &key, 0);
bpf_map_delete_elem(bpf_map__fd(skel->maps.hash2), &key);
ASSERT_EQ(skel->bss->pass2, 2, "pass2 == 2");
err = bpf_obj_get_info_by_fd(bpf_program__fd(skel->progs.on_lookup),
err = bpf_obj_get_info_by_fd(bpf_program__fd(skel->progs.on_delete),
&prog_info, &prog_info_len);
if (!ASSERT_OK(err, "get_prog_info"))
goto out;

4
tools/testing/selftests/bpf/prog_tests/select_reuseport.c
View File

@@ -114,7 +114,7 @@ static int prepare_bpf_obj(void)
err = bpf_object__load(obj);
RET_ERR(err, "load bpf_object", "err:%d\n", err);
prog = bpf_program__next(NULL, obj);
prog = bpf_object__next_program(obj, NULL);
RET_ERR(!prog, "get first bpf_program", "!prog\n");
select_by_skb_data_prog = bpf_program__fd(prog);
RET_ERR(select_by_skb_data_prog < 0, "get prog fd",
@@ -858,7 +858,7 @@ out:
cleanup();
}
void test_select_reuseport(void)
void serial_test_select_reuseport(void)
{
saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL);
if (saved_tcp_fo < 0)

3
tools/testing/selftests/bpf/prog_tests/send_signal_sched_switch.c
View File

@@ -25,7 +25,8 @@ static void *worker(void *p)
return NULL;
}
void test_send_signal_sched_switch(void)
/* NOTE: cause events loss */
void serial_test_send_signal_sched_switch(void)
{
struct test_send_signal_kern *skel;
pthread_t threads[THREAD_COUNT];

2
tools/testing/selftests/bpf/prog_tests/sk_storage_tracing.c
View File

@@ -105,7 +105,7 @@ out:
close(listen_fd);
}
void test_sk_storage_tracing(void)
void serial_test_sk_storage_tracing(void)
{
struct test_sk_storage_trace_itself *skel_itself;
int err;

2
tools/testing/selftests/bpf/prog_tests/snprintf_btf.c
View File

@@ -6,7 +6,7 @@
/* Demonstrate that bpf_snprintf_btf succeeds and that various data types
* are formatted correctly.
*/
void test_snprintf_btf(void)
void serial_test_snprintf_btf(void)
{
struct netif_receive_skb *skel;
struct netif_receive_skb__bss *bss;

2
tools/testing/selftests/bpf/prog_tests/sock_fields.c
View File

@@ -329,7 +329,7 @@ done:
close(listen_fd);
}
void test_sock_fields(void)
void serial_test_sock_fields(void)
{
struct bpf_link *egress_link = NULL, *ingress_link = NULL;
int parent_cg_fd = -1, child_cg_fd = -1;

2
tools/testing/selftests/bpf/prog_tests/sockmap_listen.c
View File

@@ -2002,7 +2002,7 @@ static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map,
test_udp_unix_redir(skel, map, family);
}
void test_sockmap_listen(void)
void serial_test_sockmap_listen(void)
{
struct test_sockmap_listen *skel;

2
tools/testing/selftests/bpf/prog_tests/tcp_rtt.c
View File

@@ -109,7 +109,7 @@ static int run_test(int cgroup_fd, int server_fd)
return -1;
}
map = bpf_map__next(NULL, obj);
map = bpf_object__next_map(obj, NULL);
map_fd = bpf_map__fd(map);
err = bpf_prog_attach(prog_fd, cgroup_fd, BPF_CGROUP_SOCK_OPS, 0);

3
tools/testing/selftests/bpf/prog_tests/timer.c
View File

@@ -39,7 +39,8 @@ static int timer(struct timer *timer_skel)
return 0;
}
void test_timer(void)
/* TODO: use pid filtering */
void serial_test_timer(void)
{
struct timer *timer_skel = NULL;
int err;

2
tools/testing/selftests/bpf/prog_tests/timer_mim.c
View File

@@ -52,7 +52,7 @@ static int timer_mim(struct timer_mim *timer_skel)
return 0;
}
void test_timer_mim(void)
void serial_test_timer_mim(void)
{
struct timer_mim_reject *timer_reject_skel = NULL;
libbpf_print_fn_t old_print_fn = NULL;

2
tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
View File

@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
void test_tp_attach_query(void)
void serial_test_tp_attach_query(void)
{
const int num_progs = 3;
int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];

2
tools/testing/selftests/bpf/prog_tests/trace_printk.c
View File

@@ -8,7 +8,7 @@
#define TRACEBUF "/sys/kernel/debug/tracing/trace_pipe"
#define SEARCHMSG "testing,testing"
void test_trace_printk(void)
void serial_test_trace_printk(void)
{
int err = 0, iter = 0, found = 0;
struct trace_printk__bss *bss;

Some files were not shown because too many files have changed in this diff Show More