From b1debb369c4bbe597bf03daeef938acc27bd2346 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Fri, 4 Oct 2019 09:14:05 -0700 Subject: [PATCH] ANDROID: arm64: add __va_function and __pa_function We use non-canonical CFI jump tables with CONFIG_CFI_CLANG, which means the compiler replaces function address references with the address of the function's CFI jump table entry. This results in __pa_symbol(function), for example, returning the physical address of the jump table entry, which can lead to address space confusion since the jump table itself points to a virtual address. The same issue happens when passing function pointers to hypervisor code running at EL2. This change adds __va_function and __pa_function macros, which use inline assembly to take the actual function address instead, and changes the relevant code to use these macros. Bug: 145210207 Change-Id: Ie3079c10427bde705a2244cfb3cb5fb954e5e065 Signed-off-by: Sami Tolvanen --- arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/include/asm/memory.h | 18 ++++++++++++++++++ arch/arm64/include/asm/mmu_context.h | 2 +- arch/arm64/kernel/cpu-reset.h | 2 +- arch/arm64/kernel/cpufeature.c | 2 +- arch/arm64/kernel/psci.c | 3 ++- arch/arm64/kernel/smp_spin_table.c | 2 +- 7 files changed, 25 insertions(+), 6 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 54387ccd1ab2..7c63d87289c0 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -158,7 +158,7 @@ extern void *__vhe_undefined_symbol; val = lm_alias((ptr)); \ val; \ }) -#define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(kvm_nvhe_sym(sym)) +#define kvm_ksym_ref_nvhe(sym) kvm_ksym_ref(__va_function(kvm_nvhe_sym(sym))) struct kvm; struct kvm_vcpu; diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 457e8c925d04..f375f1c7918b 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -300,6 +300,24 @@ static inline void *phys_to_virt(phys_addr_t x) #define virt_to_pfn(x) __phys_to_pfn(__virt_to_phys((unsigned long)(x))) #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) +/* + * With non-canonical CFI jump tables, the compiler replaces function + * address references with the address of the function's CFI jump + * table entry. This results in __pa_symbol(function) returning the + * physical address of the jump table entry, which can lead to address + * space confusion since the jump table points to the function's + * virtual address. Therefore, use inline assembly to ensure we are + * always taking the address of the actual function. + */ +#define __va_function(x) ({ \ + void *addr; \ + asm("adrp %0, " __stringify(x) "\n\t" \ + "add %0, %0, :lo12:" __stringify(x) : "=r" (addr)); \ + addr; \ +}) + +#define __pa_function(x) __pa_symbol(__va_function(x)) + /* * virt_to_page(x) convert a _valid_ virtual address to struct page * * virt_addr_valid(x) indicates whether a virtual address is valid diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index a06974cde514..bad7401e6976 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -158,7 +158,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp) ttbr1 |= TTBR_CNP_BIT; } - replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); + replace_phys = (void *)__pa_function(idmap_cpu_replace_ttbr1); cpu_install_idmap(); replace_phys(ttbr1); diff --git a/arch/arm64/kernel/cpu-reset.h b/arch/arm64/kernel/cpu-reset.h index 054285f8d858..a05bda363272 100644 --- a/arch/arm64/kernel/cpu-reset.h +++ b/arch/arm64/kernel/cpu-reset.h @@ -22,7 +22,7 @@ static inline void __noreturn __nocfi cpu_soft_restart(unsigned long entry, unsigned long el2_switch = !is_kernel_in_hyp_mode() && is_hyp_mode_available(); - restart = (void *)__pa_symbol(__cpu_soft_restart); + restart = (void *)__pa_function(__cpu_soft_restart); cpu_install_idmap(); restart(el2_switch, entry, arg0, arg1, arg2); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6b6a1c5c51dd..1bb22289455e 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1415,7 +1415,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) if (arm64_use_ng_mappings) return; - remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); + remap_fn = (void *)__pa_function(idmap_kpti_install_ng_mappings); cpu_install_idmap(); remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir)); diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 62d2bda7adb8..bfb1a6f8282d 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -38,7 +38,8 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu) static int cpu_psci_cpu_boot(unsigned int cpu) { - int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa_symbol(secondary_entry)); + int err = psci_ops.cpu_on(cpu_logical_map(cpu), + __pa_function(secondary_entry)); if (err) pr_err("failed to boot CPU%d (%d)\n", cpu, err); diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 056772c26098..a80ff9092e86 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -88,7 +88,7 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu) * boot-loader's endianness before jumping. This is mandated by * the boot protocol. */ - writeq_relaxed(__pa_symbol(secondary_holding_pen), release_addr); + writeq_relaxed(__pa_function(secondary_holding_pen), release_addr); __flush_dcache_area((__force void *)release_addr, sizeof(*release_addr));