diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 12aa0ccc3b3d..3a2055044968 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -81,6 +81,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, + __KVM_HOST_SMCCC_FUNC___pkvm_iommu_driver_init, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 36daec0bbb0f..ea7e80bdb8a9 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -390,6 +390,12 @@ int kvm_s2mpu_init(void); static inline int kvm_s2mpu_init(void) { return -ENODEV; } #endif +enum pkvm_iommu_driver_id { + PKVM_IOMMU_NR_DRIVERS, +}; + +int pkvm_iommu_driver_init(enum pkvm_iommu_driver_id drv_id, void *data, size_t size); + struct vcpu_reset_state { unsigned long pc; unsigned long r0; diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 4c343db80324..f3e041cfe751 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -131,18 +131,5 @@ extern u64 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val); extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); extern bool kvm_nvhe_sym(smccc_trng_available); -struct kvm_iommu_ops { - int (*init)(void); - bool (*host_smc_handler)(struct kvm_cpu_context *host_ctxt); - bool (*host_mmio_dabt_handler)(struct kvm_cpu_context *host_ctxt, - phys_addr_t fault_pa, unsigned int len, - bool is_write, int rd); - void (*host_stage2_set_owner)(phys_addr_t addr, size_t size, u32 owner_id); - int (*host_stage2_adjust_mmio_range)(phys_addr_t addr, phys_addr_t *start, - phys_addr_t *end); -}; - -extern struct kvm_iommu_ops kvm_iommu_ops; -extern const struct kvm_iommu_ops kvm_s2mpu_ops; #endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index ccec383ad673..7da3971e6f29 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -14,7 +14,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ inject_fault.o va_layout.o handle_exit.o \ guest.o debug.o reset.o sys_regs.o stacktrace.o \ vgic-sys-reg-v3.o fpsimd.o pkvm.o \ - arch_timer.o trng.o vmid.o \ + arch_timer.o trng.o vmid.o iommu.o \ vgic/vgic.o vgic/vgic-init.o \ vgic/vgic-irqfd.o vgic/vgic-v2.o \ vgic/vgic-v3.o vgic/vgic-v4.o \ diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h new file mode 100644 index 000000000000..b0a27f2ee9b7 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ARM64_KVM_NVHE_IOMMU_H__ +#define __ARM64_KVM_NVHE_IOMMU_H__ + +#include +#include + +#include + +struct pkvm_iommu_ops { + /* + * Global driver initialization called before devices are registered. + * Driver-specific arguments are passed in a buffer shared by the host. + * The buffer memory has been pinned in EL2 but host retains R/W access. + * Extra care must be taken when reading from it to avoid TOCTOU bugs. + * Driver initialization lock held during callback. + */ + int (*init)(void *data, size_t size); +}; + +int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t size); + +struct kvm_iommu_ops { + int (*init)(void); + bool (*host_smc_handler)(struct kvm_cpu_context *host_ctxt); + bool (*host_mmio_dabt_handler)(struct kvm_cpu_context *host_ctxt, + phys_addr_t fault_pa, unsigned int len, + bool is_write, int rd); + void (*host_stage2_set_owner)(phys_addr_t addr, size_t size, + enum pkvm_component_id owner_id); + int (*host_stage2_adjust_mmio_range)(phys_addr_t addr, phys_addr_t *start, + phys_addr_t *end); +}; + +extern struct kvm_iommu_ops kvm_iommu_ops; +extern const struct kvm_iommu_ops kvm_s2mpu_ops; + +#endif /* __ARM64_KVM_NVHE_IOMMU_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 7086c6134331..7772aedd9b2b 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -22,7 +22,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs)) hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ - cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o + cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o iommu.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o hyp-obj-$(CONFIG_DEBUG_LIST) += list_debug.o diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 9e4dd62c1c58..e758e81c3eb1 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -1113,6 +1114,15 @@ static void handle___pkvm_teardown_vm(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_teardown_vm(handle); } +static void handle___pkvm_iommu_driver_init(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(enum pkvm_iommu_driver_id, id, host_ctxt, 1); + DECLARE_REG(void *, data, host_ctxt, 2); + DECLARE_REG(size_t, size, host_ctxt, 3); + + cpu_reg(host_ctxt, 1) = __pkvm_iommu_driver_init(id, data, size); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1147,6 +1157,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), HANDLE_FUNC(__pkvm_vcpu_sync_state), + HANDLE_FUNC(__pkvm_iommu_driver_init), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) diff --git a/arch/arm64/kvm/hyp/nvhe/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu.c new file mode 100644 index 000000000000..3bf76be92821 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/iommu.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Google LLC + * Author: David Brazdil + */ + +#include + +#include +#include +#include + +#include + +enum { + IOMMU_DRIVER_NOT_READY = 0, + IOMMU_DRIVER_INITIALIZING, + IOMMU_DRIVER_READY, +}; + +struct pkvm_iommu_driver { + const struct pkvm_iommu_ops *ops; + atomic_t state; +}; + +static struct pkvm_iommu_driver iommu_drivers[PKVM_IOMMU_NR_DRIVERS]; + +/* + * Find IOMMU driver by its ID. The input ID is treated as unstrusted + * and is properly validated. + */ +static inline struct pkvm_iommu_driver *get_driver(enum pkvm_iommu_driver_id id) +{ + size_t index = (size_t)id; + + if (index >= ARRAY_SIZE(iommu_drivers)) + return NULL; + + return &iommu_drivers[index]; +} + +static const struct pkvm_iommu_ops *get_driver_ops(enum pkvm_iommu_driver_id id) +{ + switch (id) { + default: + return NULL; + } +} + +static inline bool driver_acquire_init(struct pkvm_iommu_driver *drv) +{ + return atomic_cmpxchg_acquire(&drv->state, IOMMU_DRIVER_NOT_READY, + IOMMU_DRIVER_INITIALIZING) + == IOMMU_DRIVER_NOT_READY; +} + +static inline void driver_release_init(struct pkvm_iommu_driver *drv, + bool success) +{ + atomic_set_release(&drv->state, success ? IOMMU_DRIVER_READY + : IOMMU_DRIVER_NOT_READY); +} + +/* + * Initialize EL2 IOMMU driver. + * + * This is a common hypercall for driver initialization. Driver-specific + * arguments are passed in a shared memory buffer. The driver is expected to + * initialize it's page-table bookkeeping. + */ +int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t size) +{ + struct pkvm_iommu_driver *drv; + const struct pkvm_iommu_ops *ops; + int ret = 0; + + data = kern_hyp_va(data); + + drv = get_driver(id); + ops = get_driver_ops(id); + if (!drv || !ops) + return -EINVAL; + + if (!driver_acquire_init(drv)) + return -EBUSY; + + drv->ops = ops; + + /* This can change stage-2 mappings. */ + if (ops->init) { + ret = hyp_pin_shared_mem(data, data + size); + if (!ret) { + ret = ops->init(data, size); + hyp_unpin_shared_mem(data, data + size); + } + if (ret) + goto out; + } + +out: + driver_release_init(drv, /*success=*/!ret); + return ret; +} diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c index 6fa701ded195..f10326b7bf8b 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c @@ -13,6 +13,7 @@ #include +#include #include #include #include @@ -254,7 +255,8 @@ static void set_mpt_range_locked(struct mpt *mpt, phys_addr_t first_byte, __range_invalidation(dev, first_byte, last_byte); } -static void s2mpu_host_stage2_set_owner(phys_addr_t addr, size_t size, u32 owner_id) +static void s2mpu_host_stage2_set_owner(phys_addr_t addr, size_t size, + enum pkvm_component_id owner_id) { /* Grant access only to the default owner of the page table (ID=0). */ enum mpt_prot prot = owner_id ? MPT_PROT_NONE : MPT_PROT_RW; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index b1a02b996526..3faf77a513d9 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 13312c484fc3..cae960251048 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/arm64/kvm/iommu.c b/arch/arm64/kvm/iommu.c new file mode 100644 index 000000000000..edd7316bd61b --- /dev/null +++ b/arch/arm64/kvm/iommu.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 - Google LLC + * Author: David Brazdil + */ + +#include + +int pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t size) +{ + return kvm_call_hyp_nvhe(__pkvm_iommu_driver_init, id, data, size); +}