From 7d6a5a0eabb4ab3351c4a3ddc96a975dae65d5aa Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Fri, 4 Oct 2019 09:14:05 -0700 Subject: [PATCH] ANDROID: arm64: add __pa_function We use non-canonical CFI jump tables with CONFIG_CFI_CLANG, which means the compiler replaces function address references with the address of the function's CFI jump table entry. This results in __pa_symbol(function) returning the physical address of the jump table entry, which can lead to address space confusion since the jump table points to a virtual address. This change adds a __pa_function macro, which uses inline assembly to take the actual function address instead. Bug: 145210207 Change-Id: I674e5ed386b282a7ed32eeb1f070fb39b5c4b19c Signed-off-by: Sami Tolvanen --- arch/arm64/include/asm/memory.h | 16 ++++++++++++++++ arch/arm64/include/asm/mmu_context.h | 2 +- arch/arm64/kernel/cpu-reset.h | 2 +- arch/arm64/kernel/cpufeature.c | 2 +- arch/arm64/kernel/psci.c | 3 ++- arch/arm64/kernel/smp_spin_table.c | 2 +- 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index c23c47360664..b66f0631d30a 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -309,6 +309,22 @@ static inline void *phys_to_virt(phys_addr_t x) #define virt_to_pfn(x) __phys_to_pfn(__virt_to_phys((unsigned long)(x))) #define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x)) +/* + * With non-canonical CFI jump tables, the compiler replaces function + * address references with the address of the function's CFI jump + * table entry. This results in __pa_symbol(function) returning the + * physical address of the jump table entry, which can lead to address + * space confusion since the jump table points to the function's + * virtual address. Therefore, use inline assembly to ensure we are + * always taking the address of the actual function. + */ +#define __pa_function(x) ({ \ + unsigned long addr; \ + asm("adrp %0, " __stringify(x) "\n\t" \ + "add %0, %0, :lo12:" __stringify(x) : "=r" (addr)); \ + __pa_symbol(addr); \ +}) + /* * virt_to_page(x) convert a _valid_ virtual address to struct page * * virt_addr_valid(x) indicates whether a virtual address is valid diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 2de2b464d991..1a6f5102b865 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -156,7 +156,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp) ttbr1 |= TTBR_CNP_BIT; } - replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); + replace_phys = (void *)__pa_function(idmap_cpu_replace_ttbr1); cpu_install_idmap(); replace_phys(ttbr1); diff --git a/arch/arm64/kernel/cpu-reset.h b/arch/arm64/kernel/cpu-reset.h index 054285f8d858..a05bda363272 100644 --- a/arch/arm64/kernel/cpu-reset.h +++ b/arch/arm64/kernel/cpu-reset.h @@ -22,7 +22,7 @@ static inline void __noreturn __nocfi cpu_soft_restart(unsigned long entry, unsigned long el2_switch = !is_kernel_in_hyp_mode() && is_hyp_mode_available(); - restart = (void *)__pa_symbol(__cpu_soft_restart); + restart = (void *)__pa_function(__cpu_soft_restart); cpu_install_idmap(); restart(el2_switch, entry, arg0, arg1, arg2); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index b7db6e25a197..336f95fafe03 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1053,7 +1053,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) if (kpti_applied || kaslr_offset() > 0) return; - remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); + remap_fn = (void *)__pa_function(idmap_kpti_install_ng_mappings); cpu_install_idmap(); remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir)); diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index c9f72b2665f1..86b32e9eae80 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -38,7 +38,8 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu) static int cpu_psci_cpu_boot(unsigned int cpu) { - int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa_symbol(secondary_entry)); + int err = psci_ops.cpu_on(cpu_logical_map(cpu), + __pa_function(secondary_entry)); if (err) pr_err("failed to boot CPU%d (%d)\n", cpu, err); diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index c8a3fee00c11..d6c818333419 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -88,7 +88,7 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu) * boot-loader's endianess before jumping. This is mandated by * the boot protocol. */ - writeq_relaxed(__pa_symbol(secondary_holding_pen), release_addr); + writeq_relaxed(__pa_function(secondary_holding_pen), release_addr); __flush_dcache_area((__force void *)release_addr, sizeof(*release_addr));