ANDROID: KVM: arm64: iommu: Add pkvm_iommu_finalize

Add new hypercall that the host can use to inform the hypervisor that
all hypervisor-controlled IOMMUs have been registered and no new
registrations should be allowed. This will typically be called at the
end of kernel module initialization phase.

Bug: 190463801
Signed-off-by: David Brazdil <dbrazdil@google.com>
Change-Id: I8c175310d5b262a67947443c5a0154056a8ebf3e

arch/arm64/include/asm/kvm_asm.h

@@ -82,6 +82,7 @@ enum __kvm_host_smccc_func {
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_driver_init,
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_register,
 	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_pm_notify,
+	__KVM_HOST_SMCCC_FUNC___pkvm_iommu_finalize,
 };
 
 #define DECLARE_KVM_VHE_SYM(sym)	extern char sym[]

arch/arm64/include/asm/kvm_host.h

@@ -394,6 +394,8 @@ int pkvm_iommu_resume(struct device *dev);
 int pkvm_iommu_s2mpu_register(struct device *dev, phys_addr_t pa);
 int pkvm_iommu_sysmmu_sync_register(struct device *dev, phys_addr_t pa,
 				    struct device *parent);
+/* Reject future calls to pkvm_iommu_driver_init() and pkvm_iommu_register(). */
+int pkvm_iommu_finalize(void);
 
 struct vcpu_reset_state {
 	unsigned long	pc;

arch/arm64/kvm/hyp/include/nvhe/iommu.h

@@ -84,6 +84,7 @@ int __pkvm_iommu_register(unsigned long dev_id,
 			  void *kern_mem_va, size_t mem_size);
 int __pkvm_iommu_pm_notify(unsigned long dev_id,
 			   enum pkvm_iommu_pm_event event);
+int __pkvm_iommu_finalize(void);
 int pkvm_iommu_host_stage2_adjust_range(phys_addr_t addr, phys_addr_t *start,
 					phys_addr_t *end);
 bool pkvm_iommu_host_dabt_handler(struct kvm_cpu_context *host_ctxt, u32 esr,

arch/arm64/kvm/hyp/nvhe/hyp-main.c

@@ -1008,6 +1008,11 @@ static void handle___pkvm_iommu_pm_notify(struct kvm_cpu_context *host_ctxt)
 	cpu_reg(host_ctxt, 1) = __pkvm_iommu_pm_notify(dev_id, event);
 }
 
+static void handle___pkvm_iommu_finalize(struct kvm_cpu_context *host_ctxt)
+{
+	cpu_reg(host_ctxt, 1) = __pkvm_iommu_finalize();
+}
+
 typedef void (*hcall_t)(struct kvm_cpu_context *);
 
 #define HANDLE_FUNC(x)	[__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -1044,6 +1049,7 @@ static const hcall_t host_hcall[] = {
 	HANDLE_FUNC(__pkvm_iommu_driver_init),
 	HANDLE_FUNC(__pkvm_iommu_register),
 	HANDLE_FUNC(__pkvm_iommu_pm_notify),
+	HANDLE_FUNC(__pkvm_iommu_finalize),
 };
 
 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)

arch/arm64/kvm/hyp/nvhe/iommu.c

@@ -31,6 +31,9 @@ static struct pkvm_iommu_driver iommu_drivers[PKVM_IOMMU_NR_DRIVERS];
 /* IOMMU device list. Must only be accessed with host_kvm.lock held. */
 static LIST_HEAD(iommu_list);
 
+static bool iommu_finalized;
+static DEFINE_HYP_SPINLOCK(iommu_registration_lock);
+
 static void *iommu_mem_pool;
 static size_t iommu_mem_remaining;
 
@@ -251,13 +254,24 @@ int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t si
 
 	data = kern_hyp_va(data);
 
+	/* New driver initialization not allowed after __pkvm_iommu_finalize(). */
+	hyp_spin_lock(&iommu_registration_lock);
+	if (iommu_finalized) {
+		ret = -EPERM;
+		goto out_unlock;
+	}
+
 	drv = get_driver(id);
 	ops = get_driver_ops(id);
-	if (!drv || !ops)
-		return -EINVAL;
+	if (!drv || !ops) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
 
-	if (!driver_acquire_init(drv))
-		return -EBUSY;
+	if (!driver_acquire_init(drv)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
 
 	drv->ops = ops;
 
@@ -269,7 +283,7 @@ int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t si
 			hyp_unpin_shared_mem(data, data + size);
 		}
 		if (ret)
-			goto out;
+			goto out_release;
 	}
 
 	/*
@@ -282,9 +296,12 @@ int __pkvm_iommu_driver_init(enum pkvm_iommu_driver_id id, void *data, size_t si
 		driver_release_init(drv, /*success=*/true);
 	host_unlock_component();
 
-out:
+out_release:
 	if (ret)
 		driver_release_init(drv, /*success=*/false);
+
+out_unlock:
+	hyp_spin_unlock(&iommu_registration_lock);
 	return ret;
 }
 
@@ -299,15 +316,28 @@ int __pkvm_iommu_register(unsigned long dev_id,
 	void *mem_va = NULL;
 	int ret = 0;
 
+	/* New device registration not allowed after __pkvm_iommu_finalize(). */
+	hyp_spin_lock(&iommu_registration_lock);
+	if (iommu_finalized) {
+		ret = -EPERM;
+		goto out_unlock;
+	}
+
 	drv = get_driver(drv_id);
-	if (!drv || !is_driver_ready(drv))
-		return -ENOENT;
+	if (!drv || !is_driver_ready(drv)) {
+		ret = -ENOENT;
+		goto out_unlock;
+	}
 
-	if (!PAGE_ALIGNED(dev_pa) || !PAGE_ALIGNED(dev_size))
-		return -EINVAL;
+	if (!PAGE_ALIGNED(dev_pa) || !PAGE_ALIGNED(dev_size)) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
 
-	if (!is_mmio_range(dev_pa, dev_size))
-		return -EINVAL;
+	if (!is_mmio_range(dev_pa, dev_size)) {
+		ret = -EINVAL;
+		goto out_unlock;
+	}
 
 	/*
 	 * Accept memory donation if the host is providing new memory.
@@ -316,13 +346,15 @@ int __pkvm_iommu_register(unsigned long dev_id,
 	if (kern_mem_va && mem_size) {
 		mem_va = kern_hyp_va(kern_mem_va);
 
-		if (!PAGE_ALIGNED(mem_va) || !PAGE_ALIGNED(mem_size))
-			return -EINVAL;
+		if (!PAGE_ALIGNED(mem_va) || !PAGE_ALIGNED(mem_size)) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
 
 		ret = __pkvm_host_donate_hyp(hyp_virt_to_pfn(mem_va),
 					     mem_size >> PAGE_SHIFT);
 		if (ret)
-			return ret;
+			goto out_unlock;
 	}
 
 	host_lock_component();
@@ -331,7 +363,7 @@ int __pkvm_iommu_register(unsigned long dev_id,
 	dev = alloc_iommu(drv, mem_va, mem_size);
 	if (!dev) {
 		ret = -ENOMEM;
-		goto out;
+		goto out_free;
 	}
 
 	/* Populate the new device entry. */
@@ -345,27 +377,27 @@ int __pkvm_iommu_register(unsigned long dev_id,
 
 	if (!validate_against_existing_iommus(dev)) {
 		ret = -EBUSY;
-		goto out;
+		goto out_free;
 	}
 
 	if (parent_id) {
 		dev->parent = find_iommu_by_id(parent_id);
 		if (!dev->parent) {
 			ret = -EINVAL;
-			goto out;
+			goto out_free;
 		}
 
 		if (dev->parent->ops->validate_child) {
 			ret = dev->parent->ops->validate_child(dev->parent, dev);
 			if (ret)
-				goto out;
+				goto out_free;
 		}
 	}
 
 	if (dev->ops->validate) {
 		ret = dev->ops->validate(dev);
 		if (ret)
-			goto out;
+			goto out_free;
 	}
 
 	/*
@@ -375,14 +407,14 @@ int __pkvm_iommu_register(unsigned long dev_id,
 	 */
 	ret = host_stage2_unmap_dev_locked(dev_pa, dev_size);
 	if (ret)
-		goto out;
+		goto out_free;
 
 	/* Create EL2 mapping for the device. Do it last as it is irreversible. */
 	dev->va = (void *)__pkvm_create_private_mapping(dev_pa, dev_size,
 							PAGE_HYP_DEVICE);
 	if (IS_ERR(dev->va)) {
 		ret = PTR_ERR(dev->va);
-		goto out;
+		goto out_free;
 	}
 
 	/* Register device and prevent host from mapping the MMIO range. */
@@ -390,10 +422,26 @@ int __pkvm_iommu_register(unsigned long dev_id,
 	if (dev->parent)
 		list_add_tail(&dev->siblings, &dev->parent->children);
 
-out:
+out_free:
 	if (ret)
 		free_iommu(drv, dev);
 	host_unlock_component();
+
+out_unlock:
+	hyp_spin_unlock(&iommu_registration_lock);
+	return ret;
+}
+
+int __pkvm_iommu_finalize(void)
+{
+	int ret = 0;
+
+	hyp_spin_lock(&iommu_registration_lock);
+	if (!iommu_finalized)
+		iommu_finalized = true;
+	else
+		ret = -EPERM;
+	hyp_spin_unlock(&iommu_registration_lock);
 	return ret;
 }
 

arch/arm64/kvm/iommu.c

@@ -55,3 +55,9 @@ int pkvm_iommu_resume(struct device *dev)
 				 PKVM_IOMMU_PM_RESUME);
 }
 EXPORT_SYMBOL_GPL(pkvm_iommu_resume);
+
+int pkvm_iommu_finalize(void)
+{
+	return kvm_call_hyp_nvhe(__pkvm_iommu_finalize);
+}
+EXPORT_SYMBOL_GPL(pkvm_iommu_finalize);