ANDROID: KVM: arm64: Resolve hyp module addresses using ELF sections

Resolving the addresses of the hypervisor sections within a loadable
module using symbol assignment is fragile, particularly in the face of
mergeable sections (i.e. those emitted with SHF_MERGE by the compiler).

Instead, parse the ELF .hyp.* sections directly and remove the need for
global symbols in the hypervisor module linker script.

Signed-off-by: Will Deacon <will@kernel.org>
[will: Preserve function_nocfi() calls which aren't needed in android14-6.1]
Signed-off-by: Will Deacon <willdeacon@google.com>
Bug: 261855285
Change-Id: I91d88e1a341b91ffe52ffc770dddc9b46ccb3aa4

arch/arm64/Kconfig

@@ -141,6 +141,7 @@ config ARM64
 	select GENERIC_VDSO_TIME_NS
 	select HANDLE_DOMAIN_IRQ
 	select HARDIRQS_SW_RESEND
+	select HAVE_MOD_ARCH_SPECIFIC if (ARM64_MODULE_PLTS || KVM)
 	select HAVE_MOVE_PMD
 	select HAVE_MOVE_PUD
 	select HAVE_PCI
@@ -1933,7 +1934,6 @@ config ARM64_SVE
 config ARM64_MODULE_PLTS
 	bool "Use PLTs to allow module memory to spill over into vmalloc area"
 	depends on MODULES
-	select HAVE_MOD_ARCH_SPECIFIC
 	help
 	  Allocate PLTs when loading modules so that jumps and calls whose
 	  targets are too far away for their relative offsets to be encoded

arch/arm64/include/asm/kvm_mmu.h

@@ -122,9 +122,6 @@ void kvm_update_va_mask(struct alt_instr *alt,
 			__le32 *origptr, __le32 *updptr, int nr_inst);
 void kvm_compute_layout(void);
 void kvm_apply_hyp_relocations(void);
-void kvm_apply_hyp_module_relocations(void *mod_start, void *hyp_va,
-				      kvm_nvhe_reloc_t *begin,
-				      kvm_nvhe_reloc_t *end);
 
 #define __hyp_pa(x) (((phys_addr_t)(x)) + hyp_physvirt_offset)
 

arch/arm64/include/asm/kvm_pkvm_module.h

@@ -14,6 +14,7 @@ enum pkvm_psci_notification {
 	PKVM_PSCI_CPU_ENTRY,
 };
 
+#ifdef CONFIG_MODULES
 struct pkvm_module_ops {
 	int (*create_private_mapping)(phys_addr_t phys, size_t size,
 				      enum kvm_pgtable_prot prot,
@@ -41,28 +42,24 @@ struct pkvm_module_ops {
 	unsigned long (*kern_hyp_va)(unsigned long x);
 };
 
-struct pkvm_module_section {
+int __pkvm_load_el2_module(struct module *this, unsigned long *token);
-	void *start;
-	void *end;
-};
-
-typedef s32 kvm_nvhe_reloc_t;
-
-struct pkvm_el2_module {
-	struct pkvm_module_section text;
-	struct pkvm_module_section bss;
-	struct pkvm_module_section rodata;
-	struct pkvm_module_section data;
-	kvm_nvhe_reloc_t *relocs;
-	unsigned int nr_relocs;
-	int (*init)(const struct pkvm_module_ops *ops);
-};
-
-int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this,
-			   unsigned long *token);
 
 int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token,
 			     unsigned long hyp_text_kern_va);
+#else
+static inline int __pkvm_load_el2_module(struct module *this,
+					 unsigned long *token)
+{
+	return -ENOSYS;
+}
+
+static inline int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token,
+					   unsigned long hyp_text_kern_va)
+{
+	return -ENOSYS;
+}
+#endif /* CONFIG_MODULES */
+
 #ifdef MODULE
 /*
  * function_nocfi() does not work with function pointers, hence the macro in
@@ -70,39 +67,14 @@ int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token,
  */
 #define pkvm_load_el2_module(init_fn, token)				\
 ({									\
-	extern char __kvm_nvhe___hypmod_text_start[];			\
+	THIS_MODULE->arch.hyp.init = function_nocfi(init_fn);		\
-	extern char __kvm_nvhe___hypmod_text_end[];			\
+	__pkvm_load_el2_module(THIS_MODULE, token);			\
-	extern char __kvm_nvhe___hypmod_bss_start[];			\
-	extern char __kvm_nvhe___hypmod_bss_end[];			\
-	extern char __kvm_nvhe___hypmod_rodata_start[];			\
-	extern char __kvm_nvhe___hypmod_rodata_end[];			\
-	extern char __kvm_nvhe___hypmod_data_start[];			\
-	extern char __kvm_nvhe___hypmod_data_end[];			\
-	extern char __kvm_nvhe___hyprel_start[];			\
-	extern char __kvm_nvhe___hyprel_end[];				\
-	struct pkvm_el2_module mod;					\
-									\
-	mod.text.start		= __kvm_nvhe___hypmod_text_start;	\
-	mod.text.end		= __kvm_nvhe___hypmod_text_end;		\
-	mod.bss.start		= __kvm_nvhe___hypmod_bss_start;	\
-	mod.bss.end		= __kvm_nvhe___hypmod_bss_end;		\
-	mod.rodata.start	= __kvm_nvhe___hypmod_rodata_start;	\
-	mod.rodata.end		= __kvm_nvhe___hypmod_rodata_end;	\
-	mod.data.start		= __kvm_nvhe___hypmod_data_start;	\
-	mod.data.end		= __kvm_nvhe___hypmod_data_end;		\
-	mod.relocs		= (kvm_nvhe_reloc_t *)__kvm_nvhe___hyprel_start; \
-	mod.nr_relocs		= (__kvm_nvhe___hyprel_end - __kvm_nvhe___hyprel_start) / \
-				  sizeof(*mod.relocs); \
-	mod.init = function_nocfi(init_fn);				\
-									\
-	__pkvm_load_el2_module(&mod, THIS_MODULE, token);		\
 })
 
 #define pkvm_register_el2_mod_call(hfn, token)				\
 ({									\
-	extern char __kvm_nvhe___hypmod_text_start[];			\
+	unsigned long hyp_text_kern_va;					\
-	unsigned long hyp_text_kern_va = 				\
+	hyp_text_kern_va = THIS_MODULE->arch.hyp.text.start;	 	\
-		(unsigned long)__kvm_nvhe___hypmod_text_start;		\
 	__pkvm_register_el2_call(function_nocfi(hfn), token,		\
 				 hyp_text_kern_va);			\
 })

arch/arm64/include/asm/module.h

@@ -14,12 +14,50 @@ struct mod_plt_sec {
 	int			plt_max_entries;
 };
 
-struct mod_arch_specific {
+#define ARM64_MODULE_PLTS_ARCHDATA					\
-	struct mod_plt_sec	core;
+	struct mod_plt_sec	core;					\
-	struct mod_plt_sec	init;
+	struct mod_plt_sec	init;					\
-
+									\
-	/* for CONFIG_DYNAMIC_FTRACE */
+	/* for CONFIG_DYNAMIC_FTRACE */					\
 	struct plt_entry	*ftrace_trampolines;
+#else
+#define ARM64_MODULE_PLTS_ARCHDATA
+#endif
+
+#ifdef CONFIG_KVM
+struct pkvm_module_section {
+	void *start;
+	void *end;
+};
+
+typedef s32 kvm_nvhe_reloc_t;
+struct pkvm_module_ops;
+
+struct pkvm_el2_module {
+	struct pkvm_module_section text;
+	struct pkvm_module_section bss;
+	struct pkvm_module_section rodata;
+	struct pkvm_module_section data;
+	kvm_nvhe_reloc_t *relocs;
+	unsigned int nr_relocs;
+	int (*init)(const struct pkvm_module_ops *ops);
+};
+
+void kvm_apply_hyp_module_relocations(void *mod_start, void *hyp_va,
+				      kvm_nvhe_reloc_t *begin,
+				      kvm_nvhe_reloc_t *end);
+
+#define ARM64_MODULE_KVM_ARCHDATA					\
+	/* For pKVM hypervisor modules */				\
+	struct pkvm_el2_module	hyp;
+#else
+#define ARM64_MODULE_KVM_ARCHDATA
+#endif
+
+#ifdef CONFIG_HAVE_MOD_ARCH_SPECIFIC
+struct mod_arch_specific {
+	ARM64_MODULE_PLTS_ARCHDATA
+	ARM64_MODULE_KVM_ARCHDATA
 };
 #endif
 

arch/arm64/kernel/module.c

@@ -520,14 +520,72 @@ static int module_init_ftrace_plt(const Elf_Ehdr *hdr,
 	return 0;
 }
 
+static int module_init_hyp(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
+			   struct module *mod)
+{
+#ifdef CONFIG_KVM
+	const Elf_Shdr *s;
+
+	s = find_section(hdr, sechdrs, ".hyp.text");
+	if (!s)
+		return -ENOEXEC;
+
+	mod->arch.hyp.text = (struct pkvm_module_section) {
+		.start	= (void *)s->sh_addr,
+		.end	= (void *)s->sh_addr + s->sh_size,
+	};
+
+	s = find_section(hdr, sechdrs, ".hyp.bss");
+	if (!s)
+		return -ENOEXEC;
+
+	mod->arch.hyp.bss = (struct pkvm_module_section) {
+		.start	= (void *)s->sh_addr,
+		.end	= (void *)s->sh_addr + s->sh_size,
+	};
+
+	s = find_section(hdr, sechdrs, ".hyp.rodata");
+	if (!s)
+		return -ENOEXEC;
+
+	mod->arch.hyp.rodata = (struct pkvm_module_section) {
+		.start	= (void *)s->sh_addr,
+		.end	= (void *)s->sh_addr + s->sh_size,
+	};
+
+	s = find_section(hdr, sechdrs, ".hyp.data");
+	if (!s)
+		return -ENOEXEC;
+
+	mod->arch.hyp.data = (struct pkvm_module_section) {
+		.start	= (void *)s->sh_addr,
+		.end	= (void *)s->sh_addr + s->sh_size,
+	};
+
+	s = find_section(hdr, sechdrs, ".hyp.reloc");
+	if (!s)
+		return -ENOEXEC;
+
+	mod->arch.hyp.relocs = (void *)s->sh_addr;
+	mod->arch.hyp.nr_relocs = s->sh_size / sizeof(mod->arch.hyp.relocs);
+#endif
+	return 0;
+}
+
 int module_finalize(const Elf_Ehdr *hdr,
 		    const Elf_Shdr *sechdrs,
 		    struct module *me)
 {
+	int err;
 	const Elf_Shdr *s;
+
 	s = find_section(hdr, sechdrs, ".altinstructions");
 	if (s)
 		apply_alternatives_module((void *)s->sh_addr, s->sh_size);
 
-	return module_init_ftrace_plt(hdr, sechdrs, me);
+	err = module_init_ftrace_plt(hdr, sechdrs, me);
+	if (err)
+		return err;
+
+	return module_init_hyp(hdr, sechdrs, me);
 }

arch/arm64/kvm/hyp/nvhe/module.lds.S

@@ -6,32 +6,21 @@
 SECTIONS {
 	.hyp.text : {
 		HYP_SECTION_SYMBOL_NAME(.text) = .;
-		__hypmod_text_start = .;
 		*(.text .text.*)
-		__hypmod_text_end = .;
 	}
 
 	.hyp.bss : {
 		HYP_SECTION_SYMBOL_NAME(.bss) = .;
-		__hypmod_bss_start = .;
 		*(.bss .bss.*)
-		FILL(0)
-		__hypmod_bss_end = .;
 	}
 
 	.hyp.rodata : {
 		HYP_SECTION_SYMBOL_NAME(.rodata) = .;
-		__hypmod_rodata_start = .;
+		*(.rodata .rodata.*)
-		*(.rodata .rodata.* .note.gnu.property)
-		BYTE(0)
-		__hypmod_rodata_end = .;
 	}
 
 	.hyp.data : {
 		HYP_SECTION_SYMBOL_NAME(.data) = .;
-		__hypmod_data_start = .;
 		*(.data .data.*)
-		BYTE(0)
-		__hypmod_data_end = .;
 	}
 }

arch/arm64/kvm/hyp/nvhe/psci-relay.c

@@ -31,10 +31,12 @@ static void pkvm_psci_notify(enum pkvm_psci_notification notif, struct kvm_cpu_c
 		pkvm_psci_notifier(notif, host_ctxt);
 }
 
+#ifdef CONFIG_MODULES
 int __pkvm_register_psci_notifier(void (*cb)(enum pkvm_psci_notification, struct kvm_cpu_context *))
 {
 	return cmpxchg(&pkvm_psci_notifier, NULL, cb) ? -EBUSY : 0;
 }
+#endif
 
 #define INVALID_CPU_ID	UINT_MAX
 

arch/arm64/kvm/pkvm.c

@@ -507,7 +507,6 @@ static int __init early_pkvm_enable_modules(char *arg)
 	return 0;
 }
 early_param("kvm-arm.protected_modules", early_pkvm_enable_modules);
-#endif
 
 struct pkvm_mod_sec_mapping {
 	struct pkvm_module_section *sec;
@@ -584,9 +583,9 @@ static int __pkvm_cmp_mod_sec(const void *p1, const void *p2)
 	return s1->sec->start < s2->sec->start ? -1 : s1->sec->start > s2->sec->start;
 }
 
-int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this,
+int __pkvm_load_el2_module(struct module *this, unsigned long *token)
-			   unsigned long *token)
 {
+	struct pkvm_el2_module *mod = &this->arch.hyp;
 	struct pkvm_mod_sec_mapping secs_map[] = {
 		{ &mod->text, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_X },
 		{ &mod->bss, KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W },
@@ -616,7 +615,7 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this,
 	sort(secs_map, ARRAY_SIZE(secs_map), sizeof(secs_map[0]), __pkvm_cmp_mod_sec, NULL);
 	start = secs_map[0].sec->start;
 	end = secs_map[ARRAY_SIZE(secs_map) - 1].sec->end;
-	size = PAGE_ALIGN(end - start);
+	size = end - start;
 
 	hyp_va = (void *)kvm_call_hyp_nvhe(__pkvm_alloc_module_va, size >> PAGE_SHIFT);
 	if (!hyp_va) {
@@ -670,3 +669,4 @@ int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token,
 	return ret;
 }
 EXPORT_SYMBOL_GPL(__pkvm_register_el2_call);
+#endif /* CONFIG_MODULES */