diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index c02eb2e76c2d..f7fd80147121 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -384,6 +384,7 @@ extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS]; enum pkvm_iommu_driver_id { PKVM_IOMMU_DRIVER_S2MPU, + PKVM_IOMMU_DRIVER_SYSMMU_SYNC, PKVM_IOMMU_NR_DRIVERS, }; @@ -399,6 +400,8 @@ int pkvm_iommu_suspend(struct device *dev); int pkvm_iommu_resume(struct device *dev); int pkvm_iommu_s2mpu_register(struct device *dev, phys_addr_t pa); +int pkvm_iommu_sysmmu_sync_register(struct device *dev, phys_addr_t pa, + struct device *parent); struct vcpu_reset_state { unsigned long pc; diff --git a/arch/arm64/include/asm/kvm_s2mpu.h b/arch/arm64/include/asm/kvm_s2mpu.h index 4d517bc1d0eb..0804ece03cfd 100644 --- a/arch/arm64/include/asm/kvm_s2mpu.h +++ b/arch/arm64/include/asm/kvm_s2mpu.h @@ -12,6 +12,10 @@ #include #define S2MPU_MMIO_SIZE SZ_64K +#define SYSMMU_SYNC_MMIO_SIZE SZ_64K +#define SYSMMU_SYNC_S2_OFFSET SZ_32K +#define SYSMMU_SYNC_S2_MMIO_SIZE (SYSMMU_SYNC_MMIO_SIZE - \ + SYSMMU_SYNC_S2_OFFSET) #define NR_VIDS 8 #define NR_CTX_IDS 8 @@ -128,6 +132,13 @@ static_assert(SMPT_GRAN <= PAGE_SIZE); #define SMPT_NUM_PAGES (SMPT_SIZE / PAGE_SIZE) #define SMPT_ORDER get_order(SMPT_SIZE) +/* SysMMU_SYNC registers, relative to SYSMMU_SYNC_S2_OFFSET. */ +#define REG_NS_SYNC_CMD 0x0 +#define REG_NS_SYNC_COMP 0x4 + +#define SYNC_CMD_SYNC BIT(0) +#define SYNC_COMP_COMPLETE BIT(0) + /* * Iterate over S2MPU gigabyte regions. Skip those that cannot be modified * (the MMIO registers are read only, with reset value MPT_PROT_NONE). diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h index e9683d314938..07fe3db958c3 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h +++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h @@ -92,5 +92,6 @@ void pkvm_iommu_host_stage2_idmap(phys_addr_t start, phys_addr_t end, enum kvm_pgtable_prot prot); extern const struct pkvm_iommu_ops pkvm_s2mpu_ops; +extern const struct pkvm_iommu_ops pkvm_sysmmu_sync_ops; #endif /* __ARM64_KVM_NVHE_IOMMU_H__ */ diff --git a/arch/arm64/kvm/hyp/nvhe/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu.c index bce13e28f1ce..0f9f6950bdce 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu.c @@ -68,6 +68,8 @@ static const struct pkvm_iommu_ops *get_driver_ops(enum pkvm_iommu_driver_id id) switch (id) { case PKVM_IOMMU_DRIVER_S2MPU: return IS_ENABLED(CONFIG_KVM_S2MPU) ? &pkvm_s2mpu_ops : NULL; + case PKVM_IOMMU_DRIVER_SYSMMU_SYNC: + return IS_ENABLED(CONFIG_KVM_S2MPU) ? &pkvm_sysmmu_sync_ops : NULL; default: return NULL; } diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c index dd9c81f15898..01cd8a97fa15 100644 --- a/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c +++ b/arch/arm64/kvm/hyp/nvhe/iommu/s2mpu.c @@ -28,6 +28,9 @@ (CONTEXT_CFG_VALID_VID_CTX_VID(ctxid, vid) \ | (((ctxid) < (nr_ctx)) ? CONTEXT_CFG_VALID_VID_CTX_VALID(ctxid) : 0)) +#define for_each_child(child, dev) \ + list_for_each_entry((child), &(dev)->children, siblings) + struct s2mpu_drv_data { u32 version; u32 context_cfg_valid_vid; @@ -155,6 +158,13 @@ static void __set_control_regs(struct pkvm_iommu *dev) writel_relaxed(ctrl0, dev->va + REG_NS_CTRL0); } +/* Poll the given SFR until its value has all bits of a given mask set. */ +static void __wait_until(void __iomem *addr, u32 mask) +{ + while ((readl_relaxed(addr) & mask) != mask) + continue; +} + /* Poll the given SFR as long as its value has all bits of a given mask set. */ static void __wait_while(void __iomem *addr, u32 mask) { @@ -164,6 +174,17 @@ static void __wait_while(void __iomem *addr, u32 mask) static void __wait_for_invalidation_complete(struct pkvm_iommu *dev) { + struct pkvm_iommu *sync; + + /* + * Wait for transactions to drain if SysMMU_SYNCs were registered. + * Assumes that they are in the same power domain as the S2MPU. + */ + for_each_child(sync, dev) { + writel_relaxed(SYNC_CMD_SYNC, sync->va + REG_NS_SYNC_CMD); + __wait_until(sync->va + REG_NS_SYNC_COMP, SYNC_COMP_COMPLETE); + } + /* Must not access SFRs while S2MPU is busy invalidating (v9 only). */ if (is_version(dev, S2MPU_VERSION_9)) { __wait_while(dev->va + REG_NS_STATUS, @@ -473,9 +494,29 @@ static int s2mpu_validate(struct pkvm_iommu *dev) return 0; } +static int s2mpu_validate_child(struct pkvm_iommu *dev, struct pkvm_iommu *child) +{ + if (child->ops != &pkvm_sysmmu_sync_ops) + return -EINVAL; + + return 0; +} + +static int sysmmu_sync_validate(struct pkvm_iommu *dev) +{ + if (dev->size != SYSMMU_SYNC_S2_MMIO_SIZE) + return -EINVAL; + + if (!dev->parent || dev->parent->ops != &pkvm_s2mpu_ops) + return -EINVAL; + + return 0; +} + const struct pkvm_iommu_ops pkvm_s2mpu_ops = (struct pkvm_iommu_ops){ .init = s2mpu_init, .validate = s2mpu_validate, + .validate_child = s2mpu_validate_child, .resume = s2mpu_resume, .suspend = s2mpu_suspend, .host_stage2_idmap_prepare = s2mpu_host_stage2_idmap_prepare, @@ -483,3 +524,7 @@ const struct pkvm_iommu_ops pkvm_s2mpu_ops = (struct pkvm_iommu_ops){ .host_dabt_handler = s2mpu_host_dabt_handler, .data_size = sizeof(struct s2mpu_drv_data), }; + +const struct pkvm_iommu_ops pkvm_sysmmu_sync_ops = (struct pkvm_iommu_ops){ + .validate = sysmmu_sync_validate, +}; diff --git a/arch/arm64/kvm/iommu/s2mpu.c b/arch/arm64/kvm/iommu/s2mpu.c index 7d989afde0fb..733451d74100 100644 --- a/arch/arm64/kvm/iommu/s2mpu.c +++ b/arch/arm64/kvm/iommu/s2mpu.c @@ -84,3 +84,37 @@ int pkvm_iommu_s2mpu_register(struct device *dev, phys_addr_t addr) addr, S2MPU_MMIO_SIZE, NULL); } EXPORT_SYMBOL_GPL(pkvm_iommu_s2mpu_register); + +static int init_sysmmu_sync_driver(void) +{ + static DEFINE_MUTEX(lock); + static bool init_done; + + int ret = 0; + + mutex_lock(&lock); + if (!init_done) { + ret = pkvm_iommu_driver_init(PKVM_IOMMU_DRIVER_SYSMMU_SYNC, NULL, 0); + init_done = !ret; + } + mutex_unlock(&lock); + return ret; +} + +int pkvm_iommu_sysmmu_sync_register(struct device *dev, phys_addr_t addr, + struct device *parent) +{ + int ret; + + if (!is_protected_kvm_enabled()) + return -ENODEV; + + ret = init_sysmmu_sync_driver(); + if (ret) + return ret; + + return pkvm_iommu_register(dev, PKVM_IOMMU_DRIVER_SYSMMU_SYNC, + addr + SYSMMU_SYNC_S2_OFFSET, + SYSMMU_SYNC_S2_MMIO_SIZE, parent); +} +EXPORT_SYMBOL_GPL(pkvm_iommu_sysmmu_sync_register);