From 631f92ce30a9ee2c3ecfabf34500fb2e0f6d93f1 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 5 Jan 2023 23:24:26 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Resolve hyp module addresses using ELF sections Resolving the addresses of the hypervisor sections within a loadable module using symbol assignment is fragile, particularly in the face of mergeable sections (i.e. those emitted with SHF_MERGE by the compiler). Instead, parse the ELF .hyp.* sections directly and remove the need for global symbols in the hypervisor module linker script. Signed-off-by: Will Deacon Signed-off-by: Will Deacon Bug: 261855285 Change-Id: I91d88e1a341b91ffe52ffc770dddc9b46ccb3aa4 --- arch/arm64/Kconfig | 2 +- arch/arm64/include/asm/kvm_mmu.h | 3 - arch/arm64/include/asm/kvm_pkvm_module.h | 71 +++++++----------------- arch/arm64/include/asm/module.h | 48 ++++++++++++++-- arch/arm64/kernel/module.c | 60 +++++++++++++++++++- arch/arm64/kvm/hyp/nvhe/module.lds.S | 13 +---- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 2 + arch/arm64/kvm/pkvm.c | 8 +-- 8 files changed, 131 insertions(+), 76 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1478a5393b36..460761d22ef8 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -144,6 +144,7 @@ config ARM64 select GENERIC_GETTIMEOFDAY select GENERIC_VDSO_TIME_NS select HARDIRQS_SW_RESEND + select HAVE_MOD_ARCH_SPECIFIC if (ARM64_MODULE_PLTS || KVM) select HAVE_MOVE_PMD select HAVE_MOVE_PUD select HAVE_PCI @@ -2049,7 +2050,6 @@ config ARM64_SME config ARM64_MODULE_PLTS bool "Use PLTs to allow module memory to spill over into vmalloc area" depends on MODULES - select HAVE_MOD_ARCH_SPECIFIC help Allocate PLTs when loading modules so that jumps and calls whose targets are too far away for their relative offsets to be encoded diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index ecbe1e623685..0f2cfa107f5d 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -122,9 +122,6 @@ void kvm_update_va_mask(struct alt_instr *alt, __le32 *origptr, __le32 *updptr, int nr_inst); void kvm_compute_layout(void); void kvm_apply_hyp_relocations(void); -void kvm_apply_hyp_module_relocations(void *mod_start, void *hyp_va, - kvm_nvhe_reloc_t *begin, - kvm_nvhe_reloc_t *end); #define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset) diff --git a/arch/arm64/include/asm/kvm_pkvm_module.h b/arch/arm64/include/asm/kvm_pkvm_module.h index 68b0fbbf2fd4..3e0ebde74e03 100644 --- a/arch/arm64/include/asm/kvm_pkvm_module.h +++ b/arch/arm64/include/asm/kvm_pkvm_module.h @@ -14,6 +14,7 @@ enum pkvm_psci_notification { PKVM_PSCI_CPU_ENTRY, }; +#ifdef CONFIG_MODULES struct pkvm_module_ops { int (*create_private_mapping)(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, @@ -35,66 +36,36 @@ struct pkvm_module_ops { int (*register_hyp_panic_notifier)(void (*cb)(struct kvm_cpu_context *host_ctxt)); }; -struct pkvm_module_section { - void *start; - void *end; -}; - -typedef s32 kvm_nvhe_reloc_t; - -struct pkvm_el2_module { - struct pkvm_module_section text; - struct pkvm_module_section bss; - struct pkvm_module_section rodata; - struct pkvm_module_section data; - kvm_nvhe_reloc_t *relocs; - unsigned int nr_relocs; - int (*init)(const struct pkvm_module_ops *ops); -}; - -int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, - unsigned long *token); +int __pkvm_load_el2_module(struct module *this, unsigned long *token); int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, unsigned long hyp_text_kern_va); +#else +static inline int __pkvm_load_el2_module(struct module *this, + unsigned long *token) +{ + return -ENOSYS; +} + +static inline int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, + unsigned long hyp_text_kern_va) +{ + return -ENOSYS; +} +#endif /* CONFIG_MODULES */ + #ifdef MODULE #define pkvm_load_el2_module(init_fn, token) \ ({ \ - extern char __kvm_nvhe___hypmod_text_start[]; \ - extern char __kvm_nvhe___hypmod_text_end[]; \ - extern char __kvm_nvhe___hypmod_bss_start[]; \ - extern char __kvm_nvhe___hypmod_bss_end[]; \ - extern char __kvm_nvhe___hypmod_rodata_start[]; \ - extern char __kvm_nvhe___hypmod_rodata_end[]; \ - extern char __kvm_nvhe___hypmod_data_start[]; \ - extern char __kvm_nvhe___hypmod_data_end[]; \ - extern char __kvm_nvhe___hyprel_start[]; \ - extern char __kvm_nvhe___hyprel_end[]; \ - struct pkvm_el2_module mod; \ - \ - mod.text.start = __kvm_nvhe___hypmod_text_start; \ - mod.text.end = __kvm_nvhe___hypmod_text_end; \ - mod.bss.start = __kvm_nvhe___hypmod_bss_start; \ - mod.bss.end = __kvm_nvhe___hypmod_bss_end; \ - mod.rodata.start = __kvm_nvhe___hypmod_rodata_start; \ - mod.rodata.end = __kvm_nvhe___hypmod_rodata_end; \ - mod.data.start = __kvm_nvhe___hypmod_data_start; \ - mod.data.end = __kvm_nvhe___hypmod_data_end; \ - mod.relocs = (kvm_nvhe_reloc_t *)__kvm_nvhe___hyprel_start; \ - mod.nr_relocs = (__kvm_nvhe___hyprel_end - __kvm_nvhe___hyprel_start) / \ - sizeof(*mod.relocs); \ - mod.init = init_fn; \ - \ - __pkvm_load_el2_module(&mod, THIS_MODULE, token); \ + THIS_MODULE->arch.hyp.init = init_fn; \ + __pkvm_load_el2_module(THIS_MODULE, token); \ }) #define pkvm_register_el2_mod_call(hfn, token) \ ({ \ - extern char __kvm_nvhe___hypmod_text_start[]; \ - unsigned long hyp_text_kern_va = \ - (unsigned long)__kvm_nvhe___hypmod_text_start; \ - __pkvm_register_el2_call(hfn, token, \ - hyp_text_kern_va); \ + unsigned long hyp_text_kern_va; \ + hyp_text_kern_va = THIS_MODULE->arch.hyp.text.start; \ + __pkvm_register_el2_call(hfn, token, hyp_text_kern_va); \ }) #define pkvm_el2_mod_call(id, ...) \ diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h index 18734fed3bdd..3a505c18343a 100644 --- a/arch/arm64/include/asm/module.h +++ b/arch/arm64/include/asm/module.h @@ -14,12 +14,50 @@ struct mod_plt_sec { int plt_max_entries; }; -struct mod_arch_specific { - struct mod_plt_sec core; - struct mod_plt_sec init; - - /* for CONFIG_DYNAMIC_FTRACE */ +#define ARM64_MODULE_PLTS_ARCHDATA \ + struct mod_plt_sec core; \ + struct mod_plt_sec init; \ + \ + /* for CONFIG_DYNAMIC_FTRACE */ \ struct plt_entry *ftrace_trampolines; +#else +#define ARM64_MODULE_PLTS_ARCHDATA +#endif + +#ifdef CONFIG_KVM +struct pkvm_module_section { + void *start; + void *end; +}; + +typedef s32 kvm_nvhe_reloc_t; +struct pkvm_module_ops; + +struct pkvm_el2_module { + struct pkvm_module_section text; + struct pkvm_module_section bss; + struct pkvm_module_section rodata; + struct pkvm_module_section data; + kvm_nvhe_reloc_t *relocs; + unsigned int nr_relocs; + int (*init)(const struct pkvm_module_ops *ops); +}; + +void kvm_apply_hyp_module_relocations(void *mod_start, void *hyp_va, + kvm_nvhe_reloc_t *begin, + kvm_nvhe_reloc_t *end); + +#define ARM64_MODULE_KVM_ARCHDATA \ + /* For pKVM hypervisor modules */ \ + struct pkvm_el2_module hyp; +#else +#define ARM64_MODULE_KVM_ARCHDATA +#endif + +#ifdef CONFIG_HAVE_MOD_ARCH_SPECIFIC +struct mod_arch_specific { + ARM64_MODULE_PLTS_ARCHDATA + ARM64_MODULE_KVM_ARCHDATA }; #endif diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 76b41e4ca9fa..1f10cd529b48 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -505,14 +505,72 @@ static int module_init_ftrace_plt(const Elf_Ehdr *hdr, return 0; } +static int module_init_hyp(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, + struct module *mod) +{ +#ifdef CONFIG_KVM + const Elf_Shdr *s; + + s = find_section(hdr, sechdrs, ".hyp.text"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.text = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.bss"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.bss = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.rodata"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.rodata = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.data"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.data = (struct pkvm_module_section) { + .start = (void *)s->sh_addr, + .end = (void *)s->sh_addr + s->sh_size, + }; + + s = find_section(hdr, sechdrs, ".hyp.reloc"); + if (!s) + return -ENOEXEC; + + mod->arch.hyp.relocs = (void *)s->sh_addr; + mod->arch.hyp.nr_relocs = s->sh_size / sizeof(mod->arch.hyp.relocs); +#endif + return 0; +} + int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) { + int err; const Elf_Shdr *s; + s = find_section(hdr, sechdrs, ".altinstructions"); if (s) apply_alternatives_module((void *)s->sh_addr, s->sh_size); - return module_init_ftrace_plt(hdr, sechdrs, me); + err = module_init_ftrace_plt(hdr, sechdrs, me); + if (err) + return err; + + return module_init_hyp(hdr, sechdrs, me); } diff --git a/arch/arm64/kvm/hyp/nvhe/module.lds.S b/arch/arm64/kvm/hyp/nvhe/module.lds.S index 696ab5408265..645080c681cd 100644 --- a/arch/arm64/kvm/hyp/nvhe/module.lds.S +++ b/arch/arm64/kvm/hyp/nvhe/module.lds.S @@ -6,32 +6,21 @@ SECTIONS { .hyp.text : { HYP_SECTION_SYMBOL_NAME(.text) = .; - __hypmod_text_start = .; *(.text .text.*) - __hypmod_text_end = .; } .hyp.bss : { HYP_SECTION_SYMBOL_NAME(.bss) = .; - __hypmod_bss_start = .; *(.bss .bss.*) - FILL(0) - __hypmod_bss_end = .; } .hyp.rodata : { HYP_SECTION_SYMBOL_NAME(.rodata) = .; - __hypmod_rodata_start = .; - *(.rodata .rodata.* .note.gnu.property) - BYTE(0) - __hypmod_rodata_end = .; + *(.rodata .rodata.*) } .hyp.data : { HYP_SECTION_SYMBOL_NAME(.data) = .; - __hypmod_data_start = .; *(.data .data.*) - BYTE(0) - __hypmod_data_end = .; } } diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index c5b439fd0066..0a10582e8607 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -31,10 +31,12 @@ static void pkvm_psci_notify(enum pkvm_psci_notification notif, struct kvm_cpu_c pkvm_psci_notifier(notif, host_ctxt); } +#ifdef CONFIG_MODULES int __pkvm_register_psci_notifier(void (*cb)(enum pkvm_psci_notification, struct kvm_cpu_context *)) { return cmpxchg(&pkvm_psci_notifier, NULL, cb) ? -EBUSY : 0; } +#endif #define INVALID_CPU_ID UINT_MAX diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 2ff0eb18a57d..57395cb8f7a9 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -507,7 +507,6 @@ static int __init early_pkvm_enable_modules(char *arg) return 0; } early_param("kvm-arm.protected_modules", early_pkvm_enable_modules); -#endif struct pkvm_mod_sec_mapping { struct pkvm_module_section *sec; @@ -584,9 +583,9 @@ static int __pkvm_cmp_mod_sec(const void *p1, const void *p2) return s1->sec->start < s2->sec->start ? -1 : s1->sec->start > s2->sec->start; } -int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, - unsigned long *token) +int __pkvm_load_el2_module(struct module *this, unsigned long *token) { + struct pkvm_el2_module *mod = &this->arch.hyp; struct pkvm_mod_sec_mapping secs_map[] = { { &mod->text, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_X }, { &mod->bss, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W }, @@ -616,7 +615,7 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, sort(secs_map, ARRAY_SIZE(secs_map), sizeof(secs_map[0]), __pkvm_cmp_mod_sec, NULL); start = secs_map[0].sec->start; end = secs_map[ARRAY_SIZE(secs_map) - 1].sec->end; - size = PAGE_ALIGN(end - start); + size = end - start; hyp_va = (void *)kvm_call_hyp_nvhe(__pkvm_alloc_module_va, size >> PAGE_SHIFT); if (!hyp_va) { @@ -670,3 +669,4 @@ int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, return ret; } EXPORT_SYMBOL_GPL(__pkvm_register_el2_call); +#endif /* CONFIG_MODULES */