From da176539ffde5b8dc650664fb8d9f7732bfd59d7 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Mon, 28 Mar 2022 11:02:34 +0100 Subject: [PATCH] ANDROID: KVM: arm64: iommu: Add pkvm_iommu_finalize Add new hypercall that the host can use to inform the hypervisor that all hypervisor-controlled IOMMUs have been registered and no new registrations should be allowed. This will typically be called at the end of kernel module initialization phase. Bug: 190463801 Change-Id: I124e6693a3ac0b1a81988640fcb9e09f08bf4ccf Signed-off-by: David Brazdil (cherry picked from commit 8fd93b0ef92202f16008d97ebcb92de367751491) Signed-off-by: Mostafa Saleh --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_host.h | 2 + arch/arm64/kvm/hyp/include/nvhe/iommu.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-main.c | 6 ++ arch/arm64/kvm/hyp/nvhe/iommu.c | 94 +++++++++++++++++++------ arch/arm64/kvm/iommu.c | 6 ++ 6 files changed, 87 insertions(+), 23 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5868c6a64f99..4de90ec33f1b 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -84,6 +84,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_iommu_driver_init, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_register, __KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify, + __KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index f7fd80147121..23d216661ad8 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -402,6 +402,8 @@ int pkvm_iommu_resume(struct device *dev); int pkvm_iommu_s2mpu_register(struct device *dev, phys_addr_t pa); int pkvm_iommu_sysmmu_sync_register(struct device *dev, phys_addr_t pa, struct device *parent); +/* Reject future calls to pkvm_iommu_driver_init() and pkvm_iommu_register(). */ +int pkvm_iommu_finalize(void); struct vcpu_reset_state { unsigned long pc; diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index 07fe3db958c3..d794f705fb1a 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -84,6 +84,7 @@ int __pkvm_iommu_register(unsigned long dev_id, void *kern_mem_va, size_t mem_size); int __pkvm_iommu_pm_notify(unsigned long dev_id, enum pkvm_iommu_pm_event event); +int __pkvm_iommu_finalize(void); int pkvm_iommu_host_stage2_adjust_range(phys_addr_t addr, phys_addr_t *start, phys_addr_t *end); bool pkvm_iommu_host_dabt_handler(struct kvm_cpu_context *host_ctxt, u32 esr, diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index b4b1936b690d..a65301e85f7c 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -1143,6 +1143,11 @@ static void handle___pkvm_iommu_pm_notify(struct kvm_cpu_context *host_ctxt) cpu_reg(host_ctxt, 1) = __pkvm_iommu_pm_notify(dev_id, event); } +static void handle___pkvm_iommu_finalize(struct kvm_cpu_context *host_ctxt) +{ + cpu_reg(host_ctxt, 1) = __pkvm_iommu_finalize(); +} + typedef void (*hcall_t)(struct kvm_cpu_context *); #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x @@ -1180,6 +1185,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_iommu_driver_init), HANDLE_FUNC(__pkvm_iommu_register), HANDLE_FUNC(__pkvm_iommu_pm_notify), + HANDLE_FUNC(__pkvm_iommu_finalize), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt) diff --git a/arch/arm64/kvm/hyp/nvhe/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu.c index 702af436277d..630b5e1a643a 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu.c @@ -31,6 +31,9 @@ static struct pkvm_iommu_driver iommu_drivers[PKVM_IOMMU_NR_DRIVERS]; /* IOMMU device list. Must only be accessed with host_mmu.lock held. */ static LIST_HEAD(iommu_list); +static bool iommu_finalized; +static DEFINE_HYP_SPINLOCK(iommu_registration_lock); + static void *iommu_mem_pool; static size_t iommu_mem_remaining; @@ -251,13 +254,24 @@ int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t si data = kern_hyp_va(data); + /* New driver initialization not allowed after __pkvm_iommu_finalize(). */ + hyp_spin_lock(&iommu_registration_lock); + if (iommu_finalized) { + ret = -EPERM; + goto out_unlock; + } + drv = get_driver(id); ops = get_driver_ops(id); - if (!drv || !ops) - return -EINVAL; + if (!drv || !ops) { + ret = -EINVAL; + goto out_unlock; + } - if (!driver_acquire_init(drv)) - return -EBUSY; + if (!driver_acquire_init(drv)) { + ret = -EBUSY; + goto out_unlock; + } drv->ops = ops; @@ -269,7 +283,7 @@ int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t si hyp_unpin_shared_mem(data, data + size); } if (ret) - goto out; + goto out_release; } /* @@ -282,9 +296,12 @@ int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t si driver_release_init(drv, /*success=*/true); host_unlock_component(); -out: +out_release: if (ret) driver_release_init(drv, /*success=*/false); + +out_unlock: + hyp_spin_unlock(&iommu_registration_lock); return ret; } @@ -299,15 +316,28 @@ int __pkvm_iommu_register(unsigned long dev_id, void *mem_va = NULL; int ret = 0; + /* New device registration not allowed after __pkvm_iommu_finalize(). */ + hyp_spin_lock(&iommu_registration_lock); + if (iommu_finalized) { + ret = -EPERM; + goto out_unlock; + } + drv = get_driver(drv_id); - if (!drv || !is_driver_ready(drv)) - return -ENOENT; + if (!drv || !is_driver_ready(drv)) { + ret = -ENOENT; + goto out_unlock; + } - if (!PAGE_ALIGNED(dev_pa) || !PAGE_ALIGNED(dev_size)) - return -EINVAL; + if (!PAGE_ALIGNED(dev_pa) || !PAGE_ALIGNED(dev_size)) { + ret = -EINVAL; + goto out_unlock; + } - if (!is_mmio_range(dev_pa, dev_size)) - return -EINVAL; + if (!is_mmio_range(dev_pa, dev_size)) { + ret = -EINVAL; + goto out_unlock; + } /* * Accept memory donation if the host is providing new memory. @@ -316,13 +346,15 @@ int __pkvm_iommu_register(unsigned long dev_id, if (kern_mem_va && mem_size) { mem_va = kern_hyp_va(kern_mem_va); - if (!PAGE_ALIGNED(mem_va) || !PAGE_ALIGNED(mem_size)) - return -EINVAL; + if (!PAGE_ALIGNED(mem_va) || !PAGE_ALIGNED(mem_size)) { + ret = -EINVAL; + goto out_unlock; + } ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn(mem_va), mem_size >> PAGE_SHIFT); if (ret) - return ret; + goto out_unlock; } host_lock_component(); @@ -331,7 +363,7 @@ int __pkvm_iommu_register(unsigned long dev_id, dev = alloc_iommu(drv, mem_va, mem_size); if (!dev) { ret = -ENOMEM; - goto out; + goto out_free; } /* Populate the new device entry. */ @@ -345,27 +377,27 @@ int __pkvm_iommu_register(unsigned long dev_id, if (!validate_against_existing_iommus(dev)) { ret = -EBUSY; - goto out; + goto out_free; } if (parent_id) { dev->parent = find_iommu_by_id(parent_id); if (!dev->parent) { ret = -EINVAL; - goto out; + goto out_free; } if (dev->parent->ops->validate_child) { ret = dev->parent->ops->validate_child(dev->parent, dev); if (ret) - goto out; + goto out_free; } } if (dev->ops->validate) { ret = dev->ops->validate(dev); if (ret) - goto out; + goto out_free; } /* @@ -375,13 +407,13 @@ int __pkvm_iommu_register(unsigned long dev_id, */ ret = host_stage2_unmap_dev_locked(dev_pa, dev_size); if (ret) - goto out; + goto out_free; /* Create EL2 mapping for the device. */ ret = __pkvm_create_private_mapping(dev_pa, dev_size, PAGE_HYP_DEVICE, (unsigned long *)(&dev->va)); if (ret){ - goto out; + goto out_free; } /* Register device and prevent host from mapping the MMIO range. */ @@ -389,10 +421,26 @@ int __pkvm_iommu_register(unsigned long dev_id, if (dev->parent) list_add_tail(&dev->siblings, &dev->parent->children); -out: +out_free: if (ret) free_iommu(drv, dev); host_unlock_component(); + +out_unlock: + hyp_spin_unlock(&iommu_registration_lock); + return ret; +} + +int __pkvm_iommu_finalize(void) +{ + int ret = 0; + + hyp_spin_lock(&iommu_registration_lock); + if (!iommu_finalized) + iommu_finalized = true; + else + ret = -EPERM; + hyp_spin_unlock(&iommu_registration_lock); return ret; } diff --git a/arch/arm64/kvm/iommu.c b/arch/arm64/kvm/iommu.c index 01176199e08f..6ca171327b28 100644 --- a/arch/arm64/kvm/iommu.c +++ b/arch/arm64/kvm/iommu.c @@ -55,3 +55,9 @@ int pkvm_iommu_resume(struct device *dev) PKVM_IOMMU_PM_RESUME); } EXPORT_SYMBOL_GPL(pkvm_iommu_resume); + +int pkvm_iommu_finalize(void) +{ + return kvm_call_hyp_nvhe(__pkvm_iommu_finalize); +} +EXPORT_SYMBOL_GPL(pkvm_iommu_finalize);