diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index 452d5de0f215..c95f1c08371f 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -29,6 +29,14 @@ struct pkvm_iommu_ops { int (*suspend)(struct pkvm_iommu *dev); int (*resume)(struct pkvm_iommu *dev); + /* + * Host data abort handler callback. Called with host lock held. + * Returns true if the data abort has been handled. + */ + bool (*host_dabt_handler)(struct pkvm_iommu *dev, + struct kvm_cpu_context *host_ctxt, + u32 esr, size_t off); + /* Amount of memory allocated per-device for use by the driver. */ size_t data_size; }; @@ -53,6 +61,8 @@ int __pkvm_iommu_pm_notify(unsigned long dev_id, enum pkvm_iommu_pm_event event); int pkvm_iommu_host_stage2_adjust_range(phys_addr_t addr, phys_addr_t *start, phys_addr_t *end); +bool pkvm_iommu_host_dabt_handler(struct kvm_cpu_context *host_ctxt, u32 esr, + phys_addr_t fault_pa); struct kvm_iommu_ops { int (*init)(void); diff --git a/arch/arm64/kvm/hyp/nvhe/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu.c index 3ad158369413..2d3d05d7ac89 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu.c @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -358,3 +359,24 @@ int pkvm_iommu_host_stage2_adjust_range(phys_addr_t addr, phys_addr_t *start, *end = new_end; return 0; } + +bool pkvm_iommu_host_dabt_handler(struct kvm_cpu_context *host_ctxt, u32 esr, + phys_addr_t pa) +{ + struct pkvm_iommu *dev; + + assert_host_component_locked(); + + list_for_each_entry(dev, &iommu_list, list) { + if (pa < dev->pa || pa >= dev->pa + dev->size) + continue; + + if (!dev->powered || !dev->ops->host_dabt_handler || + !dev->ops->host_dabt_handler(dev, host_ctxt, esr, pa - dev->pa)) + return false; + + kvm_skip_host_instr(); + return true; + } + return false; +} diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 392f862d76ff..b1db2b8fb0fc 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -630,9 +630,9 @@ static int host_stage2_idmap(u64 addr) enum kvm_pgtable_prot prot; int ret; - prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT; + hyp_assert_lock_held(&host_mmu.lock); - host_lock_component(); + prot = is_memory ? PKVM_HOST_MEM_PROT : PKVM_HOST_MMIO_PROT; /* * Adjust against IOMMU devices first. host_stage2_adjust_range() should * be called last for proper alignment. @@ -641,18 +641,14 @@ static int host_stage2_idmap(u64 addr) ret = pkvm_iommu_host_stage2_adjust_range(addr, &range.start, &range.end); if (ret) - goto unlock; + return ret; } ret = host_stage2_adjust_range(addr, &range); if (ret) - goto unlock; + return ret; - ret = host_stage2_idmap_locked(range.start, range.end - range.start, prot); -unlock: - host_unlock_component(); - - return ret; + return host_stage2_idmap_locked(range.start, range.end - range.start, prot); } static void host_inject_abort(struct kvm_cpu_context *host_ctxt) @@ -699,26 +695,6 @@ static void host_inject_abort(struct kvm_cpu_context *host_ctxt) write_sysreg_el2(spsr, SYS_SPSR); } -static int host_mmio_dabt_handler(struct kvm_cpu_context *host_ctxt, u32 esr, - phys_addr_t addr) -{ - bool wnr = esr & ESR_ELx_WNR; - unsigned int len = BIT((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT); - int rd = (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT; - bool handled = false; - - if (kvm_iommu_ops.host_mmio_dabt_handler) { - handled = kvm_iommu_ops.host_mmio_dabt_handler(host_ctxt, addr, - len, wnr, rd); - } - - if (!handled) - return -EPERM; - - kvm_skip_host_instr(); - return 0; -} - static bool is_dabt(u64 esr) { return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_LOW; @@ -736,14 +712,19 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt) addr = (fault.hpfar_el2 & HPFAR_MASK) << 8; addr |= fault.far_el2 & FAR_MASK; - /* See if any subsystem can handle this abort. */ - if (is_dabt(esr) && !addr_is_memory(addr)) - ret = host_mmio_dabt_handler(host_ctxt, esr, addr); + host_lock_component(); + + /* Check if an IOMMU device can handle the DABT. */ + if (is_dabt(esr) && !addr_is_memory(addr) && + pkvm_iommu_host_dabt_handler(host_ctxt, esr, addr)) + ret = 0; /* If not handled, attempt to map the page. */ if (ret == -EPERM) ret = host_stage2_idmap(addr); + host_unlock_component(); + if (ret == -EPERM) host_inject_abort(host_ctxt); else