ANDROID: KVM: arm64: iommu: Driver initialization hypcall

Add '__pkvm_iommu_driver_init' hypcall and 'struct pkvm_iommu_ops' with
an 'init' callback implemented by an EL2 driver. Driver-specific data
can be passed to 'init' from the host. The memory is pinned while
the callback processed it.

Bug: 190463801
Change-Id: I1185350bb46d41ff060a207af8e6d1f2f8a3d32d
Signed-off-by: David Brazdil <dbrazdil@google.com>
(cherry picked from commit 1d9ae14c92)
Signed-off-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Quentin Perret <qperret@google.com>

arch/arm64/include/asm/kvm_asm.h

@@ -81,6 +81,7 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load,
 	__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put,
 	__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state,
+	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_driver_init,
 };
 
 #define DECLARE_KVM_VHE_SYM(sym)	extern char sym[]

arch/arm64/include/asm/kvm_host.h

@@ -390,6 +390,12 @@ int kvm_s2mpu_init(void);
 static inline int kvm_s2mpu_init(void) { return -ENODEV; }
 #endif
 
+enum pkvm_iommu_driver_id {
+	PKVM_IOMMU_NR_DRIVERS,
+};
+
+int pkvm_iommu_driver_init(enum pkvm_iommu_driver_id drv_id, void *data, size_t size);
+
 struct vcpu_reset_state {
 	unsigned long	pc;
 	unsigned long	r0;

arch/arm64/include/asm/kvm_hyp.h

@@ -131,18 +131,5 @@ extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val);
 extern unsigned long kvm_nvhe_sym(__icache_flags);
 extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
 extern bool kvm_nvhe_sym(smccc_trng_available);
-struct kvm_iommu_ops {
-	int (*init)(void);
-	bool (*host_smc_handler)(struct kvm_cpu_context *host_ctxt);
-	bool (*host_mmio_dabt_handler)(struct kvm_cpu_context *host_ctxt,
-				       phys_addr_t fault_pa, unsigned int len,
-				       bool is_write, int rd);
-	void (*host_stage2_set_owner)(phys_addr_t addr, size_t size, u32 owner_id);
-	int (*host_stage2_adjust_mmio_range)(phys_addr_t addr, phys_addr_t *start,
-					     phys_addr_t *end);
-};
-
-extern struct kvm_iommu_ops kvm_iommu_ops;
-extern const struct kvm_iommu_ops kvm_s2mpu_ops;
 
 #endif /* __ARM64_KVM_HYP_H__ */

arch/arm64/kvm/Makefile

@@ -14,7 +14,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
 	 inject_fault.o va_layout.o handle_exit.o \
 	 guest.o debug.o reset.o sys_regs.o stacktrace.o \
 	 vgic-sys-reg-v3.o fpsimd.o pkvm.o \
-	 arch_timer.o trng.o vmid.o \
+	 arch_timer.o trng.o vmid.o iommu.o \
 	 vgic/vgic.o vgic/vgic-init.o \
 	 vgic/vgic-irqfd.o vgic/vgic-v2.o \
 	 vgic/vgic-v3.o vgic/vgic-v4.o \

arch/arm64/kvm/hyp/include/nvhe/iommu.h

@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef __ARM64_KVM_NVHE_IOMMU_H__
+#define __ARM64_KVM_NVHE_IOMMU_H__
+
+#include <linux/types.h>
+#include <asm/kvm_host.h>
+
+#include <nvhe/mem_protect.h>
+
+struct pkvm_iommu_ops {
+	/*
+	 * Global driver initialization called before devices are registered.
+	 * Driver-specific arguments are passed in a buffer shared by the host.
+	 * The buffer memory has been pinned in EL2 but host retains R/W access.
+	 * Extra care must be taken when reading from it to avoid TOCTOU bugs.
+	 * Driver initialization lock held during callback.
+	 */
+	int (*init)(void *data, size_t size);
+};
+
+int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t size);
+
+struct kvm_iommu_ops {
+	int (*init)(void);
+	bool (*host_smc_handler)(struct kvm_cpu_context *host_ctxt);
+	bool (*host_mmio_dabt_handler)(struct kvm_cpu_context *host_ctxt,
+				       phys_addr_t fault_pa, unsigned int len,
+				       bool is_write, int rd);
+	void (*host_stage2_set_owner)(phys_addr_t addr, size_t size,
+				      enum pkvm_component_id owner_id);
+	int (*host_stage2_adjust_mmio_range)(phys_addr_t addr, phys_addr_t *start,
+					     phys_addr_t *end);
+};
+
+extern struct kvm_iommu_ops kvm_iommu_ops;
+extern const struct kvm_iommu_ops kvm_s2mpu_ops;
+
+#endif	/* __ARM64_KVM_NVHE_IOMMU_H__ */

arch/arm64/kvm/hyp/nvhe/Makefile

@@ -22,7 +22,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs))
 
 hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
 	 hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \
-	 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
+	 cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o iommu.o
 hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
 	 ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
 hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o

arch/arm64/kvm/hyp/nvhe/hyp-main.c

@@ -16,6 +16,7 @@
 #include <asm/kvm_mmu.h>
 
 #include <nvhe/ffa.h>
+#include <nvhe/iommu.h>
 #include <nvhe/mem_protect.h>
 #include <nvhe/mm.h>
 #include <nvhe/pkvm.h>
@@ -1113,6 +1114,15 @@ static void handle___pkvm_teardown_vm(struct kvm_cpu_context *host_ctxt)
 	cpu_reg(host_ctxt, 1) = __pkvm_teardown_vm(handle);
 }
 
+static void handle___pkvm_iommu_driver_init(struct kvm_cpu_context *host_ctxt)
+{
+	DECLARE_REG(enum pkvm_iommu_driver_id, id, host_ctxt, 1);
+	DECLARE_REG(void *, data, host_ctxt, 2);
+	DECLARE_REG(size_t, size, host_ctxt, 3);
+
+	cpu_reg(host_ctxt, 1) = __pkvm_iommu_driver_init(id, data, size);
+}
+
 typedef void (*hcall_t)(struct kvm_cpu_context *);
 
 #define HANDLE_FUNC(x)	[__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -1147,6 +1157,7 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__pkvm_vcpu_load),
 	HANDLE_FUNC(__pkvm_vcpu_put),
 	HANDLE_FUNC(__pkvm_vcpu_sync_state),
+	HANDLE_FUNC(__pkvm_iommu_driver_init),
 };
 
 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)

arch/arm64/kvm/hyp/nvhe/iommu.c

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 Google LLC
+ * Author: David Brazdil <dbrazdil@google.com>
+ */
+
+#include <linux/kvm_host.h>
+
+#include <asm/kvm_asm.h>
+#include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
+
+#include <nvhe/iommu.h>
+
+enum {
+	IOMMU_DRIVER_NOT_READY = 0,
+	IOMMU_DRIVER_INITIALIZING,
+	IOMMU_DRIVER_READY,
+};
+
+struct pkvm_iommu_driver {
+	const struct pkvm_iommu_ops *ops;
+	atomic_t state;
+};
+
+static struct pkvm_iommu_driver iommu_drivers[PKVM_IOMMU_NR_DRIVERS];
+
+/*
+ * Find IOMMU driver by its ID. The input ID is treated as unstrusted
+ * and is properly validated.
+ */
+static inline struct pkvm_iommu_driver *get_driver(enum pkvm_iommu_driver_id id)
+{
+	size_t index = (size_t)id;
+
+	if (index >= ARRAY_SIZE(iommu_drivers))
+		return NULL;
+
+	return &iommu_drivers[index];
+}
+
+static const struct pkvm_iommu_ops *get_driver_ops(enum pkvm_iommu_driver_id id)
+{
+	switch (id) {
+	default:
+		return NULL;
+	}
+}
+
+static inline bool driver_acquire_init(struct pkvm_iommu_driver *drv)
+{
+	return atomic_cmpxchg_acquire(&drv->state, IOMMU_DRIVER_NOT_READY,
+				      IOMMU_DRIVER_INITIALIZING)
+			== IOMMU_DRIVER_NOT_READY;
+}
+
+static inline void driver_release_init(struct pkvm_iommu_driver *drv,
+				       bool success)
+{
+	atomic_set_release(&drv->state, success ? IOMMU_DRIVER_READY
+						: IOMMU_DRIVER_NOT_READY);
+}
+
+/*
+ * Initialize EL2 IOMMU driver.
+ *
+ * This is a common hypercall for driver initialization. Driver-specific
+ * arguments are passed in a shared memory buffer. The driver is expected to
+ * initialize it's page-table bookkeeping.
+ */
+int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t size)
+{
+	struct pkvm_iommu_driver *drv;
+	const struct pkvm_iommu_ops *ops;
+	int ret = 0;
+
+	data = kern_hyp_va(data);
+
+	drv = get_driver(id);
+	ops = get_driver_ops(id);
+	if (!drv || !ops)
+		return -EINVAL;
+
+	if (!driver_acquire_init(drv))
+		return -EBUSY;
+
+	drv->ops = ops;
+
+	/* This can change stage-2 mappings. */
+	if (ops->init) {
+		ret = hyp_pin_shared_mem(data, data + size);
+		if (!ret) {
+			ret = ops->init(data, size);
+			hyp_unpin_shared_mem(data, data + size);
+		}
+		if (ret)
+			goto out;
+	}
+
+out:
+	driver_release_init(drv, /*success=*/!ret);
+	return ret;
+}

arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c

@@ -13,6 +13,7 @@
 
 #include <linux/arm-smccc.h>
 
+#include <nvhe/iommu.h>
 #include <nvhe/memory.h>
 #include <nvhe/mm.h>
 #include <nvhe/spinlock.h>
@@ -254,7 +255,8 @@ static void set_mpt_range_locked(struct mpt *mpt, phys_addr_t first_byte,
 		__range_invalidation(dev, first_byte, last_byte);
 }
 
-static void s2mpu_host_stage2_set_owner(phys_addr_t addr, size_t size, u32 owner_id)
+static void s2mpu_host_stage2_set_owner(phys_addr_t addr, size_t size,
+					enum pkvm_component_id owner_id)
 {
 	/* Grant access only to the default owner of the page table (ID=0). */
 	enum mpt_prot prot = owner_id ? MPT_PROT_NONE : MPT_PROT_RW;

arch/arm64/kvm/hyp/nvhe/mem_protect.c

@@ -16,6 +16,7 @@
 #include <hyp/fault.h>
 
 #include <nvhe/gfp.h>
+#include <nvhe/iommu.h>
 #include <nvhe/memory.h>
 #include <nvhe/mem_protect.h>
 #include <nvhe/mm.h>

arch/arm64/kvm/hyp/nvhe/setup.c

@@ -13,6 +13,7 @@
 #include <nvhe/early_alloc.h>
 #include <nvhe/ffa.h>
 #include <nvhe/gfp.h>
+#include <nvhe/iommu.h>
 #include <nvhe/memory.h>
 #include <nvhe/mem_protect.h>
 #include <nvhe/mm.h>

arch/arm64/kvm/iommu.c

@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 - Google LLC
+ * Author: David Brazdil <dbrazdil@google.com>
+ */
+
+#include <linux/kvm_host.h>
+
+int pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t size)
+{
+	return kvm_call_hyp_nvhe(__pkvm_iommu_driver_init, id, data, size);
+}