ANDROID: KVM: arm64: Add support for custom hypercall registration

When pKVM is in use, allow pKVM modules to register custom hypercall
handlers:

   * pkvm_register_el2_call(): Give a handler to the hypervisor and gets in
     return the newly registered hypercall number.

   * pkvm_el2_mod_call(): Call the previously registered hypercall handler.

There is a limit of 128 hypercalls that can be registered.

Bug: 244543039
Bug: 244373730
Change-Id: I3d6c89675efe5f65f6b53c3b45ae155d1a00164c
Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
Signed-off-by: Quentin Perret <qperret@google.com>

arch/arm64/include/asm/kvm_asm.h

@@ -89,6 +89,13 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_register,
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify,
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize,
+	__KVM_HOST_SMCCC_FUNC___pkvm_register_hcall,
+
+	/*
+	 * Start of the dynamically registered hypercalls. Start a bit
+	 * further, just in case some modules...
+	 */
+	__KVM_HOST_SMCCC_FUNC___dynamic_hcalls = 128,
 };
 
 #define DECLARE_KVM_VHE_SYM(sym)	extern char sym[]

arch/arm64/include/asm/kvm_pkvm_module.h

@@ -6,6 +6,8 @@
 #include <asm/kvm_pgtable.h>
 #include <linux/export.h>
 
+typedef void (*dyn_hcall_t)(struct kvm_cpu_context *);
+
 struct pkvm_module_ops {
 	int (*create_private_mapping)(phys_addr_t phys, size_t size,
 				      enum kvm_pgtable_prot prot,
@@ -70,5 +72,28 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this,
 									\
 	__pkvm_load_el2_module(&mod, THIS_MODULE, token);		\
 })
+
+int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token,
+			     unsigned long hyp_text_kern_va);
+
+#define pkvm_register_el2_mod_call(hfn, token)				\
+({									\
+	extern char __kvm_nvhe___hypmod_text_start[];			\
+	unsigned long hyp_text_kern_va = 				\
+		(unsigned long)__kvm_nvhe___hypmod_text_start;		\
+	__pkvm_register_el2_call(function_nocfi(hfn), token,		\
+				 hyp_text_kern_va);			\
+})
+
+#define pkvm_el2_mod_call(id, ...)					\
+	({								\
+		struct arm_smccc_res res;				\
+									\
+		arm_smccc_1_1_hvc(KVM_HOST_SMCCC_ID(id),		\
+				  ##__VA_ARGS__, &res);			\
+		WARN_ON(res.a0 != SMCCC_RET_SUCCESS);			\
+									\
+		res.a1;							\
+	})
 #endif
 #endif

arch/arm64/kvm/hyp/include/nvhe/modules.h

@@ -1,5 +1,18 @@
+#include <asm/kvm_pgtable.h>
+
+#define HCALL_HANDLED 0
+#define HCALL_UNHANDLED -1
+
 #ifdef CONFIG_MODULES
 int __pkvm_init_module(void *module_init);
+int __pkvm_register_hcall(unsigned long hfn_hyp_va);
+int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt);
 #else
 static inline int __pkvm_init_module(void *module_init); { return -EOPNOTSUPP; }
+static inline int
+__pkvm_register_hcall(unsigned long hfn_hyp_va) { return -EOPNOTSUPP; }
+static inline int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt)
+{
+	return HCALL_UNHANDLED;
+}
 #endif

arch/arm64/kvm/hyp/nvhe/hyp-main.c

@@ -1186,6 +1186,13 @@ static void handle___pkvm_init_module(struct kvm_cpu_context *host_ctxt)
 	cpu_reg(host_ctxt, 1) = __pkvm_init_module(ptr);
 }
 
+static void handle___pkvm_register_hcall(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(unsigned long, hfn_hyp_va, host_ctxt, 1);
+
+	cpu_reg(host_ctxt, 1) = __pkvm_register_hcall(hfn_hyp_va);
+}
+
 typedef void (*hcall_t)(struct kvm_cpu_context *);
 
 #define HANDLE_FUNC(x)	[__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -1228,6 +1235,7 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__pkvm_map_module_page),
 	HANDLE_FUNC(__pkvm_unmap_module_page),
 	HANDLE_FUNC(__pkvm_init_module),
+	HANDLE_FUNC(__pkvm_register_hcall),
 };
 
 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
@@ -1236,6 +1244,9 @@ static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
 	unsigned long hcall_min = 0;
 	hcall_t hfn;
 
+	if (handle_host_dynamic_hcall(host_ctxt) == HCALL_HANDLED)
+		return;
+
 	/*
 	 * If pKVM has been initialised then reject any calls to the
 	 * early "privileged" hypercalls. Note that we cannot reject

arch/arm64/kvm/hyp/nvhe/modules.c

@@ -9,6 +9,8 @@
 #include <nvhe/modules.h>
 #include <nvhe/mm.h>
 #include <nvhe/serial.h>
+#include <nvhe/spinlock.h>
+#include <nvhe/trap_handler.h>
 
 static void __kvm_flush_dcache_to_poc(void *addr, size_t size)
 {
@@ -33,6 +35,73 @@ int __pkvm_init_module(void *module_init)
 	int ret;
 
 	ret = do_module_init(&module_ops);
-
 	return ret;
 }
+
+#define MAX_DYNAMIC_HCALLS 128
+
+atomic_t num_dynamic_hcalls = ATOMIC_INIT(0);
+DEFINE_HYP_SPINLOCK(dyn_hcall_lock);
+
+static dyn_hcall_t host_dynamic_hcalls[MAX_DYNAMIC_HCALLS];
+
+int handle_host_dynamic_hcall(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(unsigned long, id, host_ctxt, 0);
+	dyn_hcall_t hfn;
+	int dyn_id;
+
+	/*
+	 * TODO: static key to protect when no dynamic hcall is registered?
+	 */
+
+	dyn_id = (int)(id - KVM_HOST_SMCCC_ID(0)) -
+		 __KVM_HOST_SMCCC_FUNC___dynamic_hcalls;
+	if (dyn_id < 0)
+		return HCALL_UNHANDLED;
+
+	cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED;
+
+	/*
+	 * Order access to num_dynamic_hcalls and host_dynamic_hcalls. Paired
+	 * with __pkvm_register_hcall().
+	 */
+	if (dyn_id >= atomic_read_acquire(&num_dynamic_hcalls))
+		goto end;
+
+	hfn = READ_ONCE(host_dynamic_hcalls[dyn_id]);
+	if (!hfn)
+		goto end;
+
+	cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
+	hfn(host_ctxt);
+end:
+	return HCALL_HANDLED;
+}
+
+int __pkvm_register_hcall(unsigned long hvn_hyp_va)
+{
+	dyn_hcall_t hfn = (void *)hvn_hyp_va;
+	int reserved_id;
+
+	hyp_spin_lock(&dyn_hcall_lock);
+
+	reserved_id = atomic_read(&num_dynamic_hcalls);
+
+	if (reserved_id >= MAX_DYNAMIC_HCALLS) {
+		hyp_spin_unlock(&dyn_hcall_lock);
+		return -ENOMEM;
+	}
+
+	WRITE_ONCE(host_dynamic_hcalls[reserved_id], hfn);
+
+	/*
+	 * Order access to num_dynamic_hcalls and host_dynamic_hcalls. Paired
+	 * with handle_host_dynamic_hcall.
+	 */
+	atomic_set_release(&num_dynamic_hcalls, reserved_id + 1);
+
+	hyp_spin_unlock(&dyn_hcall_lock);
+
+	return reserved_id + __KVM_HOST_SMCCC_FUNC___dynamic_hcalls;
+};

arch/arm64/kvm/pkvm.c

@@ -561,3 +561,18 @@ int __pkvm_load_el2_module(struct pkvm_el2_module *mod, struct module *this,
 	return 0;
 }
 EXPORT_SYMBOL_GPL(__pkvm_load_el2_module);
+
+int __pkvm_register_el2_call(dyn_hcall_t hfn, unsigned long token,
+			     unsigned long hyp_text_kern_va)
+{
+	unsigned long hfn_hyp_va, offset, text_hyp_va = token;
+	int ret;
+
+	offset = (unsigned long)hfn - hyp_text_kern_va;
+	hfn_hyp_va = text_hyp_va + offset;
+
+	ret = kvm_call_hyp_nvhe(__pkvm_register_hcall,
+				(unsigned long)hfn_hyp_va);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__pkvm_register_el2_call);