ANDROID: gunyah: Sync with latest "gunyah: vm_mgr: Add/remove user memory regions"

Align Gunyah memory parcel to Gunyah v12 patches posted to kernel.org.

We deviate from a perfect copy from kernel.org because:
 - in pages_are_mergeable, zone_device_pages_have_same_pgmap is not
   present in 6.1. Drop this check.

https://lore.kernel.org/all/20230509204801.2824351-11-quic_eberman@quicinc.com/

Bug: 279506910
Change-Id: I90ec2ac416b24bcc65635f27cae7665ce879783f
Signed-off-by: Elliot Berman <quic_eberman@quicinc.com>

drivers/virt/gunyah/vm_mgr.c

@@ -11,7 +11,6 @@
 #include <linux/gunyah_rsc_mgr.h>
 #include <linux/gunyah_vm_mgr.h>
 #include <linux/miscdevice.h>
-#include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/xarray.h>
 
@@ -442,7 +441,6 @@ static void gh_vm_stop(struct gh_vm *ghvm)
 static void gh_vm_free(struct work_struct *work)
 {
 	struct gh_vm *ghvm = container_of(work, struct gh_vm, free_work);
-	struct gh_vm_mem *mapping, *tmp;
 	int ret;
 
 	switch (ghvm->vm_status) {
@@ -466,12 +464,7 @@ static void gh_vm_free(struct work_struct *work)
 			wait_event(ghvm->vm_status_wait, ghvm->vm_status == GH_RM_VM_STATUS_RESET);
 		}
 
-		mutex_lock(&ghvm->mm_lock);
-		list_for_each_entry_safe(mapping, tmp, &ghvm->memory_mappings, list) {
-			gh_vm_mem_reclaim(ghvm, mapping);
-			kfree(mapping);
-		}
-		mutex_unlock(&ghvm->mm_lock);
+		gh_vm_mem_reclaim(ghvm);
 		fallthrough;
 	case GH_RM_VM_STATUS_NO_STATE:
 		ret = gh_rm_dealloc_vmid(ghvm->rm, ghvm->vmid);
@@ -544,6 +537,8 @@ static __must_check struct gh_vm *gh_vm_alloc(struct gh_rm *rm)
 		return ERR_PTR(ret);
 	}
 
+	mmgrab(current->mm);
+	ghvm->mm = current->mm;
 	mutex_init(&ghvm->mm_lock);
 	INIT_LIST_HEAD(&ghvm->memory_mappings);
 	init_rwsem(&ghvm->status_lock);
@@ -586,8 +581,8 @@ static int gh_vm_start(struct gh_vm *ghvm)
 		if (ret) {
 			dev_warn(ghvm->parent, "Failed to %s parcel %d: %d\n",
 				mapping->share_type == VM_MEM_LEND ? "lend" : "share",
-				mapping->parcel.label,
-				ret);
+				mapping->parcel.label, ret);
+			mutex_unlock(&ghvm->mm_lock);
 			goto err;
 		}
 	}
@@ -711,6 +706,10 @@ static long gh_vm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	case GH_VM_SET_USER_MEM_REGION: {
 		struct gh_userspace_memory_region region;
 
+		/* only allow owner task to add memory */
+		if (ghvm->mm != current->mm)
+			return -EPERM;
+
 		if (copy_from_user(&region, argp, sizeof(region)))
 			return -EFAULT;
 
@@ -727,7 +726,6 @@ static long gh_vm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		if (copy_from_user(&dtb_config, argp, sizeof(dtb_config)))
 			return -EFAULT;
 
-		dtb_config.size = PAGE_ALIGN(dtb_config.size);
 		if (dtb_config.guest_phys_addr + dtb_config.size < dtb_config.guest_phys_addr)
 			return -EOVERFLOW;
 

drivers/virt/gunyah/vm_mgr.h

@@ -50,6 +50,7 @@ struct gh_vm {
 
 	struct work_struct free_work;
 	struct kref kref;
+	struct mm_struct *mm; /* userspace tied to this vm */
 	struct mutex mm_lock;
 	struct list_head memory_mappings;
 	struct mutex fn_lock;
@@ -62,9 +63,7 @@ struct gh_vm {
 };
 
 int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *region, bool lend);
-void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping);
-int gh_vm_mem_free(struct gh_vm *ghvm, u32 label);
-struct gh_vm_mem *gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label);
+void gh_vm_mem_reclaim(struct gh_vm *ghvm);
 struct gh_vm_mem *gh_vm_mem_find_by_addr(struct gh_vm *ghvm, u64 guest_phys_addr, u32 size);
 
 int gh_vm_mmio_write(struct gh_vm *ghvm, u64 addr, u32 len, u64 data);

drivers/virt/gunyah/vm_mgr_mm.c

@@ -12,6 +12,21 @@
 
 #include "vm_mgr.h"
 
+static bool pages_are_mergeable(struct page *a, struct page *b)
+{
+	if (page_to_pfn(a) + 1 != page_to_pfn(b))
+		return false;
+	return true;
+}
+
+static bool gh_vm_mem_overlap(struct gh_vm_mem *a, u64 addr, u64 size)
+{
+	u64 a_end = a->guest_phys_addr + (a->npages << PAGE_SHIFT);
+	u64 end = addr + size;
+
+	return a->guest_phys_addr < end && addr < a_end;
+}
+
 static struct gh_vm_mem *__gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label)
 	__must_hold(&ghvm->mm_lock)
 {
@@ -24,10 +39,10 @@ static struct gh_vm_mem *__gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label
 	return NULL;
 }
 
-void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping)
+static void gh_vm_mem_reclaim_mapping(struct gh_vm *ghvm, struct gh_vm_mem *mapping)
 	__must_hold(&ghvm->mm_lock)
 {
-	int i, ret = 0;
+	int ret = 0;
 
 	if (mapping->parcel.mem_handle != GH_MEM_HANDLE_INVAL) {
 		ret = gh_rm_mem_reclaim(ghvm->rm, &mapping->parcel);
@@ -36,9 +51,10 @@ void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping)
 				mapping->parcel.label, ret);
 	}
 
-	if (!ret)
-		for (i = 0; i < mapping->npages; i++)
-			unpin_user_page(mapping->pages[i]);
+	if (!ret) {
+		unpin_user_pages(mapping->pages, mapping->npages);
+		account_locked_vm(ghvm->mm, mapping->npages, false);
+	}
 
 	kfree(mapping->pages);
 	kfree(mapping->parcel.acl_entries);
@@ -47,21 +63,32 @@ void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping)
 	list_del(&mapping->list);
 }
 
+void gh_vm_mem_reclaim(struct gh_vm *ghvm)
+{
+	struct gh_vm_mem *mapping, *tmp;
+
+	mutex_lock(&ghvm->mm_lock);
+
+	list_for_each_entry_safe(mapping, tmp, &ghvm->memory_mappings, list) {
+		gh_vm_mem_reclaim_mapping(ghvm, mapping);
+		kfree(mapping);
+	}
+
+	mutex_unlock(&ghvm->mm_lock);
+}
+
 struct gh_vm_mem *gh_vm_mem_find_by_addr(struct gh_vm *ghvm, u64 guest_phys_addr, u32 size)
 {
-	struct gh_vm_mem *mapping = NULL;
-	int ret;
+	struct gh_vm_mem *mapping;
 
-	ret = mutex_lock_interruptible(&ghvm->mm_lock);
-	if (ret)
-		return ERR_PTR(ret);
+	if (overflows_type(guest_phys_addr + size, u64))
+		return NULL;
+
+	mutex_lock(&ghvm->mm_lock);
 
 	list_for_each_entry(mapping, &ghvm->memory_mappings, list) {
-		if (guest_phys_addr >= mapping->guest_phys_addr &&
-			(guest_phys_addr + size <= mapping->guest_phys_addr +
-			(mapping->npages << PAGE_SHIFT))) {
+		if (gh_vm_mem_overlap(mapping, guest_phys_addr, size))
 			goto unlock;
-		}
 	}
 
 	mapping = NULL;
@@ -70,91 +97,81 @@ unlock:
 	return mapping;
 }
 
-struct gh_vm_mem *gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label)
-{
-	struct gh_vm_mem *mapping;
-	int ret;
-
-	ret = mutex_lock_interruptible(&ghvm->mm_lock);
-	if (ret)
-		return ERR_PTR(ret);
-
-	mapping = __gh_vm_mem_find_by_label(ghvm, label);
-	mutex_unlock(&ghvm->mm_lock);
-
-	return mapping ? : ERR_PTR(-ENODEV);
-}
-
 int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *region, bool lend)
 {
 	struct gh_vm_mem *mapping, *tmp_mapping;
-	struct gh_rm_mem_entry *mem_entries;
-	phys_addr_t curr_page, prev_page;
+	struct page *curr_page, *prev_page;
 	struct gh_rm_mem_parcel *parcel;
 	int i, j, pinned, ret = 0;
+	unsigned int gup_flags;
 	size_t entry_size;
 	u16 vmid;
 
 	if (!region->memory_size || !PAGE_ALIGNED(region->memory_size) ||
-		!PAGE_ALIGNED(region->userspace_addr) || !PAGE_ALIGNED(region->guest_phys_addr))
+		!PAGE_ALIGNED(region->userspace_addr) ||
+		!PAGE_ALIGNED(region->guest_phys_addr))
 		return -EINVAL;
 
-	if (region->guest_phys_addr + region->memory_size < region->guest_phys_addr)
+	if (overflows_type(region->guest_phys_addr + region->memory_size, u64))
 		return -EOVERFLOW;
 
 	ret = mutex_lock_interruptible(&ghvm->mm_lock);
 	if (ret)
 		return ret;
 
+	/* Check label is unique */
 	mapping = __gh_vm_mem_find_by_label(ghvm, region->label);
 	if (mapping) {
-		mutex_unlock(&ghvm->mm_lock);
-		return -EEXIST;
+		ret = -EEXIST;
+		goto unlock;
 	}
 
-	mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
-	if (!mapping) {
-		mutex_unlock(&ghvm->mm_lock);
-		return -ENOMEM;
-	}
-
-	mapping->parcel.label = region->label;
-	mapping->guest_phys_addr = region->guest_phys_addr;
-	mapping->npages = region->memory_size >> PAGE_SHIFT;
-	parcel = &mapping->parcel;
-	parcel->mem_handle = GH_MEM_HANDLE_INVAL; /* to be filled later by mem_share/mem_lend */
-	parcel->mem_type = GH_RM_MEM_TYPE_NORMAL;
-
 	/* Check for overlap */
 	list_for_each_entry(tmp_mapping, &ghvm->memory_mappings, list) {
-		if (!((mapping->guest_phys_addr + (mapping->npages << PAGE_SHIFT) <=
-			tmp_mapping->guest_phys_addr) ||
-			(mapping->guest_phys_addr >=
-			tmp_mapping->guest_phys_addr + (tmp_mapping->npages << PAGE_SHIFT)))) {
+		if (gh_vm_mem_overlap(tmp_mapping, region->guest_phys_addr,
+					region->memory_size)) {
 			ret = -EEXIST;
-			goto free_mapping;
+			goto unlock;
 		}
 	}
 
-	list_add(&mapping->list, &ghvm->memory_mappings);
+	mapping = kzalloc(sizeof(*mapping), GFP_KERNEL_ACCOUNT);
+	if (!mapping) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
 
-	mapping->pages = kcalloc(mapping->npages, sizeof(*mapping->pages), GFP_KERNEL);
+	mapping->guest_phys_addr = region->guest_phys_addr;
+	mapping->npages = region->memory_size >> PAGE_SHIFT;
+	parcel = &mapping->parcel;
+	parcel->label = region->label;
+	parcel->mem_handle = GH_MEM_HANDLE_INVAL; /* to be filled later by mem_share/mem_lend */
+	parcel->mem_type = GH_RM_MEM_TYPE_NORMAL;
+
+	ret = account_locked_vm(ghvm->mm, mapping->npages, true);
+	if (ret)
+		goto free_mapping;
+
+	mapping->pages = kcalloc(mapping->npages, sizeof(*mapping->pages), GFP_KERNEL_ACCOUNT);
 	if (!mapping->pages) {
 		ret = -ENOMEM;
 		mapping->npages = 0; /* update npages for reclaim */
-		goto reclaim;
+		goto unlock_pages;
 	}
 
+	gup_flags = FOLL_LONGTERM;
+	if (region->flags & GH_MEM_ALLOW_WRITE)
+		gup_flags |= FOLL_WRITE;
+
 	pinned = pin_user_pages_fast(region->userspace_addr, mapping->npages,
-					FOLL_WRITE | FOLL_LONGTERM, mapping->pages);
+					gup_flags, mapping->pages);
 	if (pinned < 0) {
 		ret = pinned;
-		mapping->npages = 0; /* update npages for reclaim */
-		goto reclaim;
+		goto free_pages;
 	} else if (pinned != mapping->npages) {
 		ret = -EFAULT;
 		mapping->npages = pinned; /* update npages for reclaim */
-		goto reclaim;
+		goto unpin_pages;
 	}
 
 	if (lend) {
@@ -164,15 +181,16 @@ int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *regio
 		parcel->n_acl_entries = 2;
 		mapping->share_type = VM_MEM_SHARE;
 	}
-	parcel->acl_entries = kcalloc(parcel->n_acl_entries, sizeof(*parcel->acl_entries),
-					GFP_KERNEL);
+	parcel->acl_entries = kcalloc(parcel->n_acl_entries,
+				      sizeof(*parcel->acl_entries), GFP_KERNEL);
 	if (!parcel->acl_entries) {
 		ret = -ENOMEM;
-		goto reclaim;
+		goto unpin_pages;
 	}
 
-	parcel->acl_entries[0].vmid = cpu_to_le16(ghvm->vmid);
-
+	/* acl_entries[0].vmid will be this VM's vmid. We'll fill it when the
+	 * VM is starting and we know the VM's vmid.
+	 */
 	if (region->flags & GH_MEM_ALLOW_READ)
 		parcel->acl_entries[0].perms |= GH_RM_ACL_R;
 	if (region->flags & GH_MEM_ALLOW_WRITE)
@@ -180,78 +198,67 @@ int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *regio
 	if (region->flags & GH_MEM_ALLOW_EXEC)
 		parcel->acl_entries[0].perms |= GH_RM_ACL_X;
 
-	if (mapping->share_type == VM_MEM_SHARE) {
+	if (!lend) {
 		ret = gh_rm_get_vmid(ghvm->rm, &vmid);
 		if (ret)
-			goto reclaim;
+			goto free_acl;
 
 		parcel->acl_entries[1].vmid = cpu_to_le16(vmid);
 		/* Host assumed to have all these permissions. Gunyah will not
-		 * grant new permissions if host actually had less than RWX
-		 */
-		parcel->acl_entries[1].perms |= GH_RM_ACL_R | GH_RM_ACL_W | GH_RM_ACL_X;
+		* grant new permissions if host actually had less than RWX
+		*/
+		parcel->acl_entries[1].perms = GH_RM_ACL_R | GH_RM_ACL_W | GH_RM_ACL_X;
 	}
 
-	mem_entries = kcalloc(mapping->npages, sizeof(*mem_entries), GFP_KERNEL);
-	if (!mem_entries) {
+	parcel->n_mem_entries = 1;
+	for (i = 1; i < mapping->npages; i++) {
+		if (!pages_are_mergeable(mapping->pages[i - 1], mapping->pages[i]))
+			parcel->n_mem_entries++;
+	}
+
+	parcel->mem_entries = kcalloc(parcel->n_mem_entries,
+					sizeof(parcel->mem_entries[0]),
+					GFP_KERNEL_ACCOUNT);
+	if (!parcel->mem_entries) {
 		ret = -ENOMEM;
-		goto reclaim;
+		goto free_acl;
 	}
 
 	/* reduce number of entries by combining contiguous pages into single memory entry */
-	prev_page = page_to_phys(mapping->pages[0]);
-	mem_entries[0].ipa_base = cpu_to_le64(prev_page);
+	prev_page = mapping->pages[0];
+	parcel->mem_entries[0].ipa_base = cpu_to_le64(page_to_phys(prev_page));
 	entry_size = PAGE_SIZE;
 	for (i = 1, j = 0; i < mapping->npages; i++) {
-		curr_page = page_to_phys(mapping->pages[i]);
-		if (curr_page - prev_page == PAGE_SIZE) {
+		curr_page = mapping->pages[i];
+		if (pages_are_mergeable(prev_page, curr_page)) {
 			entry_size += PAGE_SIZE;
 		} else {
-			mem_entries[j].size = cpu_to_le64(entry_size);
+			parcel->mem_entries[j].size = cpu_to_le64(entry_size);
 			j++;
-			mem_entries[j].ipa_base = cpu_to_le64(curr_page);
+			BUG_ON(j >= parcel->n_mem_entries);
+			parcel->mem_entries[j].ipa_base =
+				cpu_to_le64(page_to_phys(curr_page));
 			entry_size = PAGE_SIZE;
 		}
 
 		prev_page = curr_page;
 	}
-	mem_entries[j].size = cpu_to_le64(entry_size);
-
-	parcel->n_mem_entries = j + 1;
-	parcel->mem_entries = kmemdup(mem_entries, sizeof(*mem_entries) * parcel->n_mem_entries,
-					GFP_KERNEL);
-	kfree(mem_entries);
-	if (!parcel->mem_entries) {
-		ret = -ENOMEM;
-		goto reclaim;
-	}
+	parcel->mem_entries[j].size = cpu_to_le64(entry_size);
 
+	list_add(&mapping->list, &ghvm->memory_mappings);
 	mutex_unlock(&ghvm->mm_lock);
 	return 0;
-reclaim:
-	gh_vm_mem_reclaim(ghvm, mapping);
+free_acl:
+	kfree(parcel->acl_entries);
+unpin_pages:
+	unpin_user_pages(mapping->pages, pinned);
+free_pages:
+	kfree(mapping->pages);
+unlock_pages:
+	account_locked_vm(ghvm->mm, mapping->npages, false);
 free_mapping:
 	kfree(mapping);
-	mutex_unlock(&ghvm->mm_lock);
-	return ret;
-}
-
-int gh_vm_mem_free(struct gh_vm *ghvm, u32 label)
-{
-	struct gh_vm_mem *mapping;
-	int ret;
-
-	ret = mutex_lock_interruptible(&ghvm->mm_lock);
-	if (ret)
-		return ret;
-
-	mapping = __gh_vm_mem_find_by_label(ghvm, label);
-	if (!mapping)
-		goto out;
-
-	gh_vm_mem_reclaim(ghvm, mapping);
-	kfree(mapping);
-out:
+unlock:
 	mutex_unlock(&ghvm->mm_lock);
 	return ret;
 }

include/linux/gunyah_rsc_mgr.h

@@ -81,16 +81,16 @@ enum gh_rm_mem_type {
 };
 
 /*
- * struct gh_rm_mem_parcel - Package info about memory to be lent/shared/donated/reclaimed
+ * struct gh_rm_mem_parcel - Info about memory to be lent/shared/donated/reclaimed
  * @mem_type: The type of memory: normal (DDR) or IO
  * @label: An client-specified identifier which can be used by the other VMs to identify the purpose
  *         of the memory parcel.
+ * @n_acl_entries: Count of the number of entries in the @acl_entries array.
  * @acl_entries: An array of access control entries. Each entry specifies a VM and what access
  *               is allowed for the memory parcel.
- * @n_acl_entries: Count of the number of entries in the `acl_entries` array.
- * @mem_entries: An list of regions to be associated with the memory parcel. Addresses should be
+ * @n_mem_entries: Count of the number of entries in the @mem_entries array.
+ * @mem_entries: An array of regions to be associated with the memory parcel. Addresses should be
  *               (intermediate) physical addresses from Linux's perspective.
- * @n_mem_entries: Count of the number of entries in the `mem_entries` array.
  * @mem_handle: On success, filled with memory handle that RM allocates for this memory parcel
  */
 struct gh_rm_mem_parcel {