From e2eb8807e6b795dfcef162f7c4a45954077401c5 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Tue, 27 Sep 2022 17:15:49 +0100 Subject: [PATCH] ANDROID: KVM: arm64: Add support for custom hypercall registration When pKVM is in use, allow pKVM modules to register custom hypercall handlers: * pkvm_register_el2_call(): Give a handler to the hypervisor and gets in return the newly registered hypercall number. * pkvm_el2_mod_call(): Call the previously registered hypercall handler. There is a limit of 128 hypercalls that can be registered. Bug: 244543039 Bug: 244373730 Change-Id: I3d6c89675efe5f65f6b53c3b45ae155d1a00164c Signed-off-by: Vincent Donnefort Signed-off-by: Quentin Perret --- arch/arm64/include/asm/kvm_asm.h | 7 +++ arch/arm64/include/asm/kvm_pkvm_module.h | 25 ++++++++ arch/arm64/kvm/hyp/include/nvhe/modules.h | 13 +++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 ++++ arch/arm64/kvm/hyp/nvhe/modules.c | 71 ++++++++++++++++++++++- arch/arm64/kvm/pkvm.c | 15 +++++ 6 files changed, 141 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index bf1ace7d3efc..b3b20a886892 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -89,6 +89,13 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_iommu_register, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize, + __KVM_HOST_SMCCC_FUNC___pkvm_register_hcall, + + /* + * Start of the dynamically registered hypercalls. Start a bit + * further, just in case some modules... + */ + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls = 128, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] diff --git a/arch/arm64/include/asm/kvm_pkvm_module.h b/arch/arm64/include/asm/kvm_pkvm_module.h index 2a8ced4b53d5..86ba74e01492 100644 --- a/arch/arm64/include/asm/kvm_pkvm_module.h +++ b/arch/arm64/include/asm/kvm_pkvm_module.h @@ -6,6 +6,8 @@ #include #include +typedef void (*dyn_hcall_t)(struct kvm_cpu_context *); + struct pkvm_module_ops { int (*create_private_mapping)(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, @@ -70,5 +72,28 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, \ __pkvm_load_el2_module(&mod, THIS_MODULE, token); \ }) + +int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, + unsigned long hyp_text_kern_va); + +#define pkvm_register_el2_mod_call(hfn, token) \ +({ \ + extern char __kvm_nvhe___hypmod_text_start[]; \ + unsigned long hyp_text_kern_va = \ + (unsigned long)__kvm_nvhe___hypmod_text_start; \ + __pkvm_register_el2_call(function_nocfi(hfn), token, \ + hyp_text_kern_va); \ +}) + +#define pkvm_el2_mod_call(id, ...) \ + ({ \ + struct arm_smccc_res res; \ + \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_ID(id), \ + ##__VA_ARGS__, &res); \ + WARN_ON(res.a0 != SMCCC_RET_SUCCESS); \ + \ + res.a1; \ + }) #endif #endif diff --git a/arch/arm64/kvm/hyp/include/nvhe/modules.h b/arch/arm64/kvm/hyp/include/nvhe/modules.h index e9e05ab58594..d4d77ea1b3f6 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/modules.h +++ b/arch/arm64/kvm/hyp/include/nvhe/modules.h @@ -1,5 +1,18 @@ +#include + +#define HCALL_HANDLED 0 +#define HCALL_UNHANDLED -1 + #ifdef CONFIG_MODULES int __pkvm_init_module(void *module_init); +int __pkvm_register_hcall(unsigned long hfn_hyp_va); +int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt); #else static inline int __pkvm_init_module(void *module_init); { return -EOPNOTSUPP; } +static inline int +__pkvm_register_hcall(unsigned long hfn_hyp_va) { return -EOPNOTSUPP; } +static inline int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt) +{ + return HCALL_UNHANDLED; +} #endif diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 5be78f35ea09..9ee73e630ec5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1186,6 +1186,13 @@ static void handle___pkvm_init_module(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_init_module(ptr); } +static void handle___pkvm_register_hcall(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, hfn_hyp_va, host_ctxt, 1); + + cpu_reg(host_ctxt, 1) = __pkvm_register_hcall(hfn_hyp_va); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1228,6 +1235,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_map_module_page), HANDLE_FUNC(__pkvm_unmap_module_page), HANDLE_FUNC(__pkvm_init_module), + HANDLE_FUNC(__pkvm_register_hcall), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) @@ -1236,6 +1244,9 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) unsigned long hcall_min = 0; hcall_t hfn; + if (handle_host_dynamic_hcall(host_ctxt) == HCALL_HANDLED) + return; + /* * If pKVM has been initialised then reject any calls to the * early "privileged" hypercalls. Note that we cannot reject diff --git a/arch/arm64/kvm/hyp/nvhe/modules.c b/arch/arm64/kvm/hyp/nvhe/modules.c index b3c6fbb452d9..ee2c963a7c90 100644 --- a/arch/arm64/kvm/hyp/nvhe/modules.c +++ b/arch/arm64/kvm/hyp/nvhe/modules.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include static void __kvm_flush_dcache_to_poc(void *addr, size_t size) { @@ -33,6 +35,73 @@ int __pkvm_init_module(void *module_init) int ret; ret = do_module_init(&module_ops); - return ret; } + +#define MAX_DYNAMIC_HCALLS 128 + +atomic_t num_dynamic_hcalls = ATOMIC_INIT(0); +DEFINE_HYP_SPINLOCK(dyn_hcall_lock); + +static dyn_hcall_t host_dynamic_hcalls[MAX_DYNAMIC_HCALLS]; + +int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(unsigned long, id, host_ctxt, 0); + dyn_hcall_t hfn; + int dyn_id; + + /* + * TODO: static key to protect when no dynamic hcall is registered? + */ + + dyn_id = (int)(id - KVM_HOST_SMCCC_ID(0)) - + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls; + if (dyn_id < 0) + return HCALL_UNHANDLED; + + cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED; + + /* + * Order access to num_dynamic_hcalls and host_dynamic_hcalls. Paired + * with __pkvm_register_hcall(). + */ + if (dyn_id >= atomic_read_acquire(&num_dynamic_hcalls)) + goto end; + + hfn = READ_ONCE(host_dynamic_hcalls[dyn_id]); + if (!hfn) + goto end; + + cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS; + hfn(host_ctxt); +end: + return HCALL_HANDLED; +} + +int __pkvm_register_hcall(unsigned long hvn_hyp_va) +{ + dyn_hcall_t hfn = (void *)hvn_hyp_va; + int reserved_id; + + hyp_spin_lock(&dyn_hcall_lock); + + reserved_id = atomic_read(&num_dynamic_hcalls); + + if (reserved_id >= MAX_DYNAMIC_HCALLS) { + hyp_spin_unlock(&dyn_hcall_lock); + return -ENOMEM; + } + + WRITE_ONCE(host_dynamic_hcalls[reserved_id], hfn); + + /* + * Order access to num_dynamic_hcalls and host_dynamic_hcalls. Paired + * with handle_host_dynamic_hcall. + */ + atomic_set_release(&num_dynamic_hcalls, reserved_id + 1); + + hyp_spin_unlock(&dyn_hcall_lock); + + return reserved_id + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls; +}; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 77af44dc0019..3587fea8e555 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -561,3 +561,18 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this, return 0; } EXPORT_SYMBOL_GPL(__pkvm_load_el2_module); + +int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token, + unsigned long hyp_text_kern_va) +{ + unsigned long hfn_hyp_va, offset, text_hyp_va = token; + int ret; + + offset = (unsigned long)hfn - hyp_text_kern_va; + hfn_hyp_va = text_hyp_va + offset; + + ret = kvm_call_hyp_nvhe(__pkvm_register_hcall, + (unsigned long)hfn_hyp_va); + return ret; +} +EXPORT_SYMBOL_GPL(__pkvm_register_el2_call);