diff --git a/Makefile b/Makefile index dad8f18c8144..e643fbd80c2c 100644 --- a/Makefile +++ b/Makefile @@ -884,25 +884,6 @@ ifdef CONFIG_LIVEPATCH KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone) endif -ifdef CONFIG_CFI_CLANG -CC_FLAGS_CFI := -fsanitize=cfi \ - -fno-sanitize-cfi-canonical-jump-tables - -ifdef CONFIG_MODULES -CC_FLAGS_CFI += -fsanitize-cfi-cross-dso -endif - -ifdef CONFIG_CFI_PERMISSIVE -CC_FLAGS_CFI += -fsanitize-recover=cfi \ - -fno-sanitize-trap=cfi -endif - -# If LTO flags are filtered out, we must also filter out CFI. -CC_FLAGS_LTO += $(CC_FLAGS_CFI) -KBUILD_CFLAGS += $(CC_FLAGS_CFI) -export CC_FLAGS_CFI -endif - ifdef CONFIG_SHADOW_CALL_STACK CC_FLAGS_SCS := -fsanitize=shadow-call-stack KBUILD_CFLAGS += $(CC_FLAGS_SCS) diff --git a/arch/Kconfig b/arch/Kconfig index 7c64ea7b936a..8cc35dc556c7 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -552,29 +552,6 @@ config SHADOW_CALL_STACK reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the stacks. -config CFI_CLANG - bool "Use Clang's Control Flow Integrity (CFI)" - depends on LTO_CLANG && KALLSYMS - help - This option enables Clang's Control Flow Integrity (CFI), which adds - runtime checking for indirect function calls. - -config CFI_CLANG_SHADOW - bool "Use CFI shadow to speed up cross-module checks" - default y - depends on CFI_CLANG - help - If you select this option, the kernel builds a fast look-up table of - CFI check functions in loaded modules to reduce overhead. - -config CFI_PERMISSIVE - bool "Use CFI in permissive mode" - depends on CFI_CLANG - help - When selected, Control Flow Integrity (CFI) violations result in a - warning instead of a kernel panic. This option is useful for finding - CFI violations during development. - config HAVE_ARCH_WITHIN_STACK_FRAMES bool help diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index e4ce269eb7e0..db600ef218d7 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -94,7 +94,6 @@ */ #ifdef CONFIG_LD_DEAD_CODE_DATA_ELIMINATION #define TEXT_MAIN .text .text.[0-9a-zA-Z_]* -#define TEXT_CFI_MAIN .text.[0-9a-zA-Z_]*.cfi #define DATA_MAIN .data .data.[0-9a-zA-Z_]* .data..LPBX* #define SDATA_MAIN .sdata .sdata.[0-9a-zA-Z_]* #define RODATA_MAIN .rodata .rodata.[0-9a-zA-Z_]* @@ -102,7 +101,6 @@ #define SBSS_MAIN .sbss .sbss.[0-9a-zA-Z_]* #else #define TEXT_MAIN .text -#define TEXT_CFI_MAIN .text.cfi #define DATA_MAIN .data #define SDATA_MAIN .sdata #define RODATA_MAIN .rodata @@ -563,7 +561,6 @@ ALIGN_FUNCTION(); \ *(.text.hot TEXT_MAIN .text.fixup .text.unlikely) \ NOINSTR_TEXT \ - *(TEXT_CFI_MAIN) \ *(.text..refcount) \ *(.ref.text) \ MEM_KEEP(init.text*) \ diff --git a/include/linux/cfi.h b/include/linux/cfi.h deleted file mode 100644 index 6ef37b35faaa..000000000000 --- a/include/linux/cfi.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Clang Control Flow Integrity (CFI) support. - * - * Copyright (C) 2019 Google LLC - */ -#ifndef _LINUX_CFI_H -#define _LINUX_CFI_H - -#include - -#ifdef CONFIG_CFI_CLANG -#ifdef CONFIG_MODULES - -typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag); - -/* Compiler-generated function in each module, and the kernel */ -#define CFI_CHECK_FN __cfi_check -#define CFI_CHECK_FN_NAME __stringify(CFI_CHECK_FN) - -extern void CFI_CHECK_FN(uint64_t id, void *ptr, void *diag); - -#ifdef CONFIG_CFI_CLANG_SHADOW -extern void cfi_module_add(struct module *mod, unsigned long min_addr, - unsigned long max_addr); - -extern void cfi_module_remove(struct module *mod, unsigned long min_addr, - unsigned long max_addr); -#else -static inline void cfi_module_add(struct module *mod, unsigned long min_addr, - unsigned long max_addr) -{ -} - -static inline void cfi_module_remove(struct module *mod, unsigned long min_addr, - unsigned long max_addr) -{ -} -#endif /* CONFIG_CFI_CLANG_SHADOW */ - -#endif /* CONFIG_MODULES */ -#endif /* CONFIG_CFI_CLANG */ - -#endif /* _LINUX_CFI_H */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 19ceabf432c6..5e55302e3bf6 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -63,5 +63,3 @@ #if __has_feature(shadow_call_stack) # define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) #endif - -#define __nocfi __attribute__((__no_sanitize__("cfi"))) diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 5b7713ea0f7e..9efb884aad55 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -227,14 +227,14 @@ struct ftrace_likely_data { # define randomized_struct_fields_end #endif -#ifndef __nocfi -# define __nocfi -#endif - #ifndef __noscs # define __noscs #endif +#ifndef __norecordmcount +# define __norecordmcount +#endif + #ifndef asm_volatile_goto #define asm_volatile_goto(x...) asm goto(x) #endif diff --git a/include/linux/init.h b/include/linux/init.h index eaabe8687ea9..212fc9e2f691 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,7 +47,7 @@ /* These are for everybody (although not all archs will actually discard it in modules) */ -#define __init __section(.init.text) __cold __latent_entropy __noinitretpoline __nocfi +#define __init __section(.init.text) __cold __latent_entropy __noinitretpoline #define __initdata __section(.init.data) #define __initconst __section(.init.rodata) #define __exitdata __section(.exit.data) diff --git a/include/linux/module.h b/include/linux/module.h index b28a9eb76e70..138ccf53f8d0 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -25,7 +25,6 @@ #include #include #include -#include #include #include @@ -379,10 +378,6 @@ struct module { const s32 *crcs; unsigned int num_syms; -#ifdef CONFIG_CFI_CLANG - cfi_check_fn cfi_check; -#endif - /* Kernel parameters. */ #ifdef CONFIG_SYSFS struct mutex param_lock; diff --git a/init/Kconfig b/init/Kconfig index c4c684850189..ee7c1aa54f2d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2268,7 +2268,7 @@ endif # MODULES config MODULES_TREE_LOOKUP def_bool y - depends on PERF_EVENTS || TRACING || CFI_CLANG + depends on PERF_EVENTS || TRACING config INIT_ALL_POSSIBLE bool diff --git a/kernel/Makefile b/kernel/Makefile index a644e3537c82..f3218bc5ec69 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -40,9 +40,6 @@ CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) # cond_syscall is currently not LTO compatible CFLAGS_sys_ni.o = $(DISABLE_LTO) -# Don't instrument error handlers -CFLAGS_REMOVE_cfi.o := $(CC_FLAGS_CFI) - obj-y += sched/ obj-y += locking/ obj-y += power/ @@ -112,7 +109,6 @@ obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ obj-$(CONFIG_KCSAN) += kcsan/ obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o -obj-$(CONFIG_CFI_CLANG) += cfi.o obj-$(CONFIG_PERF_EVENTS) += events/ diff --git a/kernel/cfi.c b/kernel/cfi.c deleted file mode 100644 index 05f589e24f68..000000000000 --- a/kernel/cfi.c +++ /dev/null @@ -1,307 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Clang Control Flow Integrity (CFI) error and slowpath handling. - * - * Copyright (C) 2019 Google LLC - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* Compiler-defined handler names */ -#ifdef CONFIG_CFI_PERMISSIVE -#define cfi_failure_handler __ubsan_handle_cfi_check_fail -#define cfi_slowpath_handler __cfi_slowpath_diag -#else /* enforcing */ -#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort -#define cfi_slowpath_handler __cfi_slowpath -#endif /* CONFIG_CFI_PERMISSIVE */ - -static inline void handle_cfi_failure(void *ptr) -{ - if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) - WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr); - else - panic("CFI failure (target: %pS)\n", ptr); -} - -#ifdef CONFIG_MODULES -#ifdef CONFIG_CFI_CLANG_SHADOW -struct shadow_range { - /* Module address range */ - unsigned long mod_min_addr; - unsigned long mod_max_addr; - /* Module page range */ - unsigned long min_page; - unsigned long max_page; -}; - -#define SHADOW_ORDER 2 -#define SHADOW_PAGES (1 << SHADOW_ORDER) -#define SHADOW_SIZE \ - ((SHADOW_PAGES * PAGE_SIZE - sizeof(struct shadow_range)) / sizeof(u16)) -#define SHADOW_INVALID 0xFFFF - -struct cfi_shadow { - /* Page range covered by the shadow */ - struct shadow_range r; - /* Page offsets to __cfi_check functions in modules */ - u16 shadow[SHADOW_SIZE]; -}; - -static DEFINE_MUTEX(shadow_update_lock); -static struct cfi_shadow __rcu *cfi_shadow __read_mostly; - -static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr) -{ - unsigned long index; - unsigned long page = ptr >> PAGE_SHIFT; - - if (unlikely(page < s->r.min_page)) - return -1; /* Outside of module area */ - - index = page - s->r.min_page; - - if (index >= SHADOW_SIZE) - return -1; /* Cannot be addressed with shadow */ - - return (int)index; -} - -static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s, - int index) -{ - if (unlikely(index < 0 || index >= SHADOW_SIZE)) - return 0; - - if (unlikely(s->shadow[index] == SHADOW_INVALID)) - return 0; - - return (s->r.min_page + s->shadow[index]) << PAGE_SHIFT; -} - -static inline unsigned long shadow_to_page(const struct cfi_shadow *s, - int index) -{ - if (unlikely(index < 0 || index >= SHADOW_SIZE)) - return 0; - - return (s->r.min_page + index) << PAGE_SHIFT; -} - -static void prepare_next_shadow(const struct cfi_shadow __rcu *prev, - struct cfi_shadow *next) -{ - int i, index, check; - - /* Mark everything invalid */ - memset(next->shadow, 0xFF, sizeof(next->shadow)); - - if (!prev) - return; /* No previous shadow */ - - /* If the base address didn't change, update is not needed */ - if (prev->r.min_page == next->r.min_page) { - memcpy(next->shadow, prev->shadow, sizeof(next->shadow)); - return; - } - - /* Convert the previous shadow to the new address range */ - for (i = 0; i < SHADOW_SIZE; ++i) { - if (prev->shadow[i] == SHADOW_INVALID) - continue; - - index = ptr_to_shadow(next, shadow_to_page(prev, i)); - if (index < 0) - continue; - - check = ptr_to_shadow(next, - shadow_to_ptr(prev, prev->shadow[i])); - if (check < 0) - continue; - - next->shadow[index] = (u16)check; - } -} - -static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod) -{ - unsigned long ptr; - unsigned long min_page_addr; - unsigned long max_page_addr; - unsigned long check = (unsigned long)mod->cfi_check; - int check_index = ptr_to_shadow(s, check); - - if (unlikely((check & PAGE_MASK) != check)) - return; /* Must be page aligned */ - - if (check_index < 0) - return; /* Module not addressable with shadow */ - - min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK; - max_page_addr = (unsigned long)mod->core_layout.base + - mod->core_layout.text_size; - max_page_addr &= PAGE_MASK; - - /* For each page, store the check function index in the shadow */ - for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) { - int index = ptr_to_shadow(s, ptr); - - if (index >= 0) { - /* Each page must only contain one module */ - WARN_ON(s->shadow[index] != SHADOW_INVALID); - s->shadow[index] = (u16)check_index; - } - } -} - -static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod) -{ - unsigned long ptr; - unsigned long min_page_addr; - unsigned long max_page_addr; - - min_page_addr = (unsigned long)mod->core_layout.base & PAGE_MASK; - max_page_addr = (unsigned long)mod->core_layout.base + - mod->core_layout.text_size; - max_page_addr &= PAGE_MASK; - - for (ptr = min_page_addr; ptr <= max_page_addr; ptr += PAGE_SIZE) { - int index = ptr_to_shadow(s, ptr); - - if (index >= 0) - s->shadow[index] = SHADOW_INVALID; - } -} - -typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *); - -static void update_shadow(struct module *mod, unsigned long min_addr, - unsigned long max_addr, update_shadow_fn fn) -{ - struct cfi_shadow *prev; - struct cfi_shadow *next = (struct cfi_shadow *) - __get_free_pages(GFP_KERNEL, SHADOW_ORDER); - - next->r.mod_min_addr = min_addr; - next->r.mod_max_addr = max_addr; - next->r.min_page = min_addr >> PAGE_SHIFT; - next->r.max_page = max_addr >> PAGE_SHIFT; - - mutex_lock(&shadow_update_lock); - prev = rcu_dereference_protected(cfi_shadow, 1); - prepare_next_shadow(prev, next); - - fn(next, mod); - set_memory_ro((unsigned long)next, SHADOW_PAGES); - rcu_assign_pointer(cfi_shadow, next); - - mutex_unlock(&shadow_update_lock); - synchronize_rcu(); - - if (prev) { - set_memory_rw((unsigned long)prev, SHADOW_PAGES); - free_pages((unsigned long)prev, SHADOW_ORDER); - } -} - -void cfi_module_add(struct module *mod, unsigned long min_addr, - unsigned long max_addr) -{ - update_shadow(mod, min_addr, max_addr, add_module_to_shadow); -} -EXPORT_SYMBOL(cfi_module_add); - -void cfi_module_remove(struct module *mod, unsigned long min_addr, - unsigned long max_addr) -{ - update_shadow(mod, min_addr, max_addr, remove_module_from_shadow); -} -EXPORT_SYMBOL(cfi_module_remove); - -static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s, - unsigned long ptr) -{ - int index; - - if (unlikely(!s)) - return NULL; /* No shadow available */ - - if (ptr < s->r.mod_min_addr || ptr > s->r.mod_max_addr) - return NULL; /* Not in a mapped module */ - - index = ptr_to_shadow(s, ptr); - if (index < 0) - return NULL; /* Cannot be addressed with shadow */ - - return (cfi_check_fn)shadow_to_ptr(s, index); -} -#endif /* CONFIG_CFI_CLANG_SHADOW */ - -static inline cfi_check_fn find_module_cfi_check(void *ptr) -{ - struct module *mod; - - preempt_disable(); - mod = __module_address((unsigned long)ptr); - preempt_enable(); - - if (mod) - return mod->cfi_check; - - return CFI_CHECK_FN; -} - -static inline cfi_check_fn find_cfi_check(void *ptr) -{ -#ifdef CONFIG_CFI_CLANG_SHADOW - cfi_check_fn f; - - if (!rcu_access_pointer(cfi_shadow)) - return CFI_CHECK_FN; /* No loaded modules */ - - /* Look up the __cfi_check function to use */ - rcu_read_lock(); - f = ptr_to_check_fn(rcu_dereference(cfi_shadow), (unsigned long)ptr); - rcu_read_unlock(); - - if (f) - return f; - - /* - * Fall back to find_module_cfi_check, which works also for a larger - * module address space, but is slower. - */ -#endif /* CONFIG_CFI_CLANG_SHADOW */ - - return find_module_cfi_check(ptr); -} - -void cfi_slowpath_handler(uint64_t id, void *ptr, void *diag) -{ - cfi_check_fn check = find_cfi_check(ptr); - - if (likely(check)) - check(id, ptr, diag); - else /* Don't allow unchecked modules */ - handle_cfi_failure(ptr); -} -EXPORT_SYMBOL(cfi_slowpath_handler); -#endif /* CONFIG_MODULES */ - -void cfi_failure_handler(void *data, void *ptr, void *vtable) -{ - handle_cfi_failure(ptr); -} -EXPORT_SYMBOL(cfi_failure_handler); - -void __cfi_check_fail(void *data, void *ptr) -{ - handle_cfi_failure(ptr); -} diff --git a/kernel/module.c b/kernel/module.c index 04b1d1384668..aa183c9ac0a2 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -2162,8 +2162,6 @@ void __weak module_arch_freeing_init(struct module *mod) { } -static void cfi_cleanup(struct module *mod); - /* Free a module, remove from lists, etc. */ static void free_module(struct module *mod) { @@ -2203,9 +2201,6 @@ static void free_module(struct module *mod) synchronize_rcu(); mutex_unlock(&module_mutex); - /* Clean up CFI for the module. */ - cfi_cleanup(mod); - /* This may be empty, but that's OK */ module_arch_freeing_init(mod); module_memfree(mod->init_layout.base); @@ -3471,8 +3466,6 @@ int __weak module_finalize(const Elf_Ehdr *hdr, return 0; } -static void cfi_init(struct module *mod); - static int post_relocation(struct module *mod, const struct load_info *info) { /* Sort exception table now relocations are done. */ @@ -3485,9 +3478,6 @@ static int post_relocation(struct module *mod, const struct load_info *info) /* Setup kallsyms-specific fields. */ add_kallsyms(mod, info); - /* Setup CFI for the module. */ - cfi_init(mod); - /* Arch-specific module finalizing. */ return module_finalize(info->hdr, info->sechdrs, mod); } @@ -4291,24 +4281,6 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, } #endif /* CONFIG_KALLSYMS */ -static void cfi_init(struct module *mod) -{ -#ifdef CONFIG_CFI_CLANG - rcu_read_lock(); - mod->cfi_check = (cfi_check_fn)find_kallsyms_symbol_value(mod, - CFI_CHECK_FN_NAME); - rcu_read_unlock(); - cfi_module_add(mod, module_addr_min, module_addr_max); -#endif -} - -static void cfi_cleanup(struct module *mod) -{ -#ifdef CONFIG_CFI_CLANG - cfi_module_remove(mod, module_addr_min, module_addr_max); -#endif -} - /* Maximum number of characters written by module_flags() */ #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)