From d7d29ac76f7efb506bcecc092641e704f791d92d Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 30 Oct 2020 10:38:20 +0900 Subject: [PATCH 1/3] percpu: reduce the number of cpu distance comparisons To build group_map[] and group_cnt[], we find out which group CPUs belong to by comparing the distance of the cpu. However, this includes cases where comparisons are not required. This patch uses a bitmap to record CPUs that is not classified in the group. CPUs that we know which group they belong to should be cleared from the bitmap. In result, we can reduce the number of unnecessary comparisons. Signed-off-by: Wonhyuk Yang Signed-off-by: Dennis Zhou [Dennis: added cpumask_clear() call and #include cpumask.h.] --- mm/percpu.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/mm/percpu.c b/mm/percpu.c index ad7a37ee74ef..80f8f885a990 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -69,6 +69,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -2669,6 +2670,7 @@ static struct pcpu_alloc_info * __init pcpu_build_alloc_info( { static int group_map[NR_CPUS] __initdata; static int group_cnt[NR_CPUS] __initdata; + static struct cpumask mask __initdata; const size_t static_size = __per_cpu_end - __per_cpu_start; int nr_groups = 1, nr_units = 0; size_t size_sum, min_unit_size, alloc_size; @@ -2681,6 +2683,7 @@ static struct pcpu_alloc_info * __init pcpu_build_alloc_info( /* this function may be called multiple times */ memset(group_map, 0, sizeof(group_map)); memset(group_cnt, 0, sizeof(group_cnt)); + cpumask_clear(&mask); /* calculate size_sum and ensure dyn_size is enough for early alloc */ size_sum = PFN_ALIGN(static_size + reserved_size + @@ -2702,24 +2705,27 @@ static struct pcpu_alloc_info * __init pcpu_build_alloc_info( upa--; max_upa = upa; + cpumask_copy(&mask, cpu_possible_mask); + /* group cpus according to their proximity */ - for_each_possible_cpu(cpu) { - group = 0; - next_group: - for_each_possible_cpu(tcpu) { - if (cpu == tcpu) - break; - if (group_map[tcpu] == group && cpu_distance_fn && - (cpu_distance_fn(cpu, tcpu) > LOCAL_DISTANCE || - cpu_distance_fn(tcpu, cpu) > LOCAL_DISTANCE)) { - group++; - nr_groups = max(nr_groups, group + 1); - goto next_group; - } - } + for (group = 0; !cpumask_empty(&mask); group++) { + /* pop the group's first cpu */ + cpu = cpumask_first(&mask); group_map[cpu] = group; group_cnt[group]++; + cpumask_clear_cpu(cpu, &mask); + + for_each_cpu(tcpu, &mask) { + if (!cpu_distance_fn || + (cpu_distance_fn(cpu, tcpu) == LOCAL_DISTANCE && + cpu_distance_fn(tcpu, cpu) == LOCAL_DISTANCE)) { + group_map[tcpu] = group; + group_cnt[group]++; + cpumask_clear_cpu(tcpu, &mask); + } + } } + nr_groups = group; /* * Wasted space is caused by a ratio imbalance of upa to group_cnt. From 258e0815e2b1706e87c0d874211097aa8a7aa52f Mon Sep 17 00:00:00 2001 From: Dennis Zhou Date: Sun, 14 Feb 2021 17:16:33 +0000 Subject: [PATCH 2/3] percpu: fix clang modpost section mismatch pcpu_build_alloc_info() is an __init function that makes a call to cpumask_clear_cpu(). With CONFIG_GCOV_PROFILE_ALL enabled, the inline heuristics are modified and such cpumask_clear_cpu() which is marked inline doesn't get inlined. Because it works on mask in __initdata, modpost throws a section mismatch error. Arnd sent a patch with the flatten attribute as an alternative [2]. I've added it to compiler_attributes.h. modpost complaint: WARNING: modpost: vmlinux.o(.text+0x735425): Section mismatch in reference from the function cpumask_clear_cpu() to the variable .init.data:pcpu_build_alloc_info.mask The function cpumask_clear_cpu() references the variable __initdata pcpu_build_alloc_info.mask. This is often because cpumask_clear_cpu lacks a __initdata annotation or the annotation of pcpu_build_alloc_info.mask is wrong. clang output: mm/percpu.c:2724:5: remark: cpumask_clear_cpu not inlined into pcpu_build_alloc_info because too costly to inline (cost=725, threshold=325) [-Rpass-missed=inline] [1] https://lore.kernel.org/linux-mm/202012220454.9F6Bkz9q-lkp@intel.com/ [2] https://lore.kernel.org/lkml/CAK8P3a2ZWfNeXKSm8K_SUhhwkor17jFo3xApLXjzfPqX0eUDUA@mail.gmail.com/ Reported-by: kernel test robot Cc: Arnd Bergmann Cc: Nick Desaulniers Signed-off-by: Dennis Zhou --- include/linux/compiler_attributes.h | 6 ++++++ mm/percpu.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index ea5e04e75845..c043b8d2b17b 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -210,6 +210,12 @@ # define fallthrough do {} while (0) /* fallthrough */ #endif +/* + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#Common-Function-Attributes + * clang: https://clang.llvm.org/docs/AttributeReference.html#flatten + */ +# define __flatten __attribute__((flatten)) + /* * Note the missing underscores. * diff --git a/mm/percpu.c b/mm/percpu.c index 80f8f885a990..6596a0a4286e 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -2663,7 +2663,7 @@ early_param("percpu_alloc", percpu_alloc_setup); * On success, pointer to the new allocation_info is returned. On * failure, ERR_PTR value is returned. */ -static struct pcpu_alloc_info * __init pcpu_build_alloc_info( +static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info( size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn) From 5cf0fd591f2e7833405bdc7ce2176c3502e8a11e Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Tue, 23 Feb 2021 12:21:58 -0800 Subject: [PATCH 3/3] Kbuild: disable TRIM_UNUSED_KSYMS option The removal of EXPORT_UNUSED_SYMBOL() in commit 367948220fce looks like (and was sold as) a no-op, but it actually had a rather serious and subtle side effect: the UNUSED_SYMBOLS option not only enabled the removed (unused) functionality, it also _disabled_ the TRIM_UNUSED_KSYMS functionality. And it turns out that TRIM_UNUSED_KSYMS is a huge time waste, and takes up a third of the kernel build time for me. For no actual upside, since no distro is likely to ever be able to enable it (because they all support external kernel modules). Rather than re-enable EXPORT_UNUSED_SYMBOL, this just disables the TRIM_UNUSED_KSYMS option by marking it broken. I'm tempted to just remove the support entirely, but maybe somebody has a use-case and can fix the behavior of it. I could have just disabled it for COMPILE_TEST, but it really smells like the TRIM_UNUSED_KSYMS option is badly done and not really useful, so this takes the more direct approach - let's see if anybody ever actually notices or complains. Cc: Miroslav Benes Cc: Emil Velikov Cc: Christoph Hellwig Cc: Jessica Yu Fixes: 367948220fce ("module: remove EXPORT_UNUSED_SYMBOL*") Signed-off-by: Linus Torvalds --- init/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/init/Kconfig b/init/Kconfig index 8a2c0e41f1bb..ba8bd5256980 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2274,6 +2274,7 @@ config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS config TRIM_UNUSED_KSYMS bool "Trim unused exported kernel symbols" + depends on BROKEN help The kernel and some modules make many symbols available for other modules to use via EXPORT_SYMBOL() and variants. Depending