From 28ecb1162adbdf7143042b7ba37dedc671218b67 Mon Sep 17 00:00:00 2001 From: Elliot Berman Date: Fri, 14 Apr 2023 16:41:21 -0700 Subject: [PATCH] ANDROID: gunyah: Sync with latest "gunyah: vm_mgr: Add/remove user memory regions" Align Gunyah memory parcel to Gunyah v12 patches posted to kernel.org. We deviate from a perfect copy from kernel.org because: - in pages_are_mergeable, zone_device_pages_have_same_pgmap is not present in 6.1. Drop this check. https://lore.kernel.org/all/20230509204801.2824351-11-quic_eberman@quicinc.com/ Bug: 279506910 Change-Id: I90ec2ac416b24bcc65635f27cae7665ce879783f Signed-off-by: Elliot Berman --- drivers/virt/gunyah/vm_mgr.c | 20 ++- drivers/virt/gunyah/vm_mgr.h | 5 +- drivers/virt/gunyah/vm_mgr_mm.c | 229 ++++++++++++++++---------------- include/linux/gunyah_rsc_mgr.h | 8 +- 4 files changed, 133 insertions(+), 129 deletions(-) diff --git a/drivers/virt/gunyah/vm_mgr.c b/drivers/virt/gunyah/vm_mgr.c index a0b067c25b56..81c1d5225cde 100644 --- a/drivers/virt/gunyah/vm_mgr.c +++ b/drivers/virt/gunyah/vm_mgr.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -442,7 +441,6 @@ static void gh_vm_stop(struct gh_vm *ghvm) static void gh_vm_free(struct work_struct *work) { struct gh_vm *ghvm = container_of(work, struct gh_vm, free_work); - struct gh_vm_mem *mapping, *tmp; int ret; switch (ghvm->vm_status) { @@ -466,12 +464,7 @@ static void gh_vm_free(struct work_struct *work) wait_event(ghvm->vm_status_wait, ghvm->vm_status == GH_RM_VM_STATUS_RESET); } - mutex_lock(&ghvm->mm_lock); - list_for_each_entry_safe(mapping, tmp, &ghvm->memory_mappings, list) { - gh_vm_mem_reclaim(ghvm, mapping); - kfree(mapping); - } - mutex_unlock(&ghvm->mm_lock); + gh_vm_mem_reclaim(ghvm); fallthrough; case GH_RM_VM_STATUS_NO_STATE: ret = gh_rm_dealloc_vmid(ghvm->rm, ghvm->vmid); @@ -544,6 +537,8 @@ static __must_check struct gh_vm *gh_vm_alloc(struct gh_rm *rm) return ERR_PTR(ret); } + mmgrab(current->mm); + ghvm->mm = current->mm; mutex_init(&ghvm->mm_lock); INIT_LIST_HEAD(&ghvm->memory_mappings); init_rwsem(&ghvm->status_lock); @@ -586,8 +581,8 @@ static int gh_vm_start(struct gh_vm *ghvm) if (ret) { dev_warn(ghvm->parent, "Failed to %s parcel %d: %d\n", mapping->share_type == VM_MEM_LEND ? "lend" : "share", - mapping->parcel.label, - ret); + mapping->parcel.label, ret); + mutex_unlock(&ghvm->mm_lock); goto err; } } @@ -711,6 +706,10 @@ static long gh_vm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) case GH_VM_SET_USER_MEM_REGION: { struct gh_userspace_memory_region region; + /* only allow owner task to add memory */ + if (ghvm->mm != current->mm) + return -EPERM; + if (copy_from_user(®ion, argp, sizeof(region))) return -EFAULT; @@ -727,7 +726,6 @@ static long gh_vm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (copy_from_user(&dtb_config, argp, sizeof(dtb_config))) return -EFAULT; - dtb_config.size = PAGE_ALIGN(dtb_config.size); if (dtb_config.guest_phys_addr + dtb_config.size < dtb_config.guest_phys_addr) return -EOVERFLOW; diff --git a/drivers/virt/gunyah/vm_mgr.h b/drivers/virt/gunyah/vm_mgr.h index d713c94744fc..9fc4e30129a7 100644 --- a/drivers/virt/gunyah/vm_mgr.h +++ b/drivers/virt/gunyah/vm_mgr.h @@ -50,6 +50,7 @@ struct gh_vm { struct work_struct free_work; struct kref kref; + struct mm_struct *mm; /* userspace tied to this vm */ struct mutex mm_lock; struct list_head memory_mappings; struct mutex fn_lock; @@ -62,9 +63,7 @@ struct gh_vm { }; int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *region, bool lend); -void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping); -int gh_vm_mem_free(struct gh_vm *ghvm, u32 label); -struct gh_vm_mem *gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label); +void gh_vm_mem_reclaim(struct gh_vm *ghvm); struct gh_vm_mem *gh_vm_mem_find_by_addr(struct gh_vm *ghvm, u64 guest_phys_addr, u32 size); int gh_vm_mmio_write(struct gh_vm *ghvm, u64 addr, u32 len, u64 data); diff --git a/drivers/virt/gunyah/vm_mgr_mm.c b/drivers/virt/gunyah/vm_mgr_mm.c index b8896ca6941b..952cc85e5d4b 100644 --- a/drivers/virt/gunyah/vm_mgr_mm.c +++ b/drivers/virt/gunyah/vm_mgr_mm.c @@ -12,6 +12,21 @@ #include "vm_mgr.h" +static bool pages_are_mergeable(struct page *a, struct page *b) +{ + if (page_to_pfn(a) + 1 != page_to_pfn(b)) + return false; + return true; +} + +static bool gh_vm_mem_overlap(struct gh_vm_mem *a, u64 addr, u64 size) +{ + u64 a_end = a->guest_phys_addr + (a->npages << PAGE_SHIFT); + u64 end = addr + size; + + return a->guest_phys_addr < end && addr < a_end; +} + static struct gh_vm_mem *__gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label) __must_hold(&ghvm->mm_lock) { @@ -24,10 +39,10 @@ static struct gh_vm_mem *__gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label return NULL; } -void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping) +static void gh_vm_mem_reclaim_mapping(struct gh_vm *ghvm, struct gh_vm_mem *mapping) __must_hold(&ghvm->mm_lock) { - int i, ret = 0; + int ret = 0; if (mapping->parcel.mem_handle != GH_MEM_HANDLE_INVAL) { ret = gh_rm_mem_reclaim(ghvm->rm, &mapping->parcel); @@ -36,9 +51,10 @@ void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping) mapping->parcel.label, ret); } - if (!ret) - for (i = 0; i < mapping->npages; i++) - unpin_user_page(mapping->pages[i]); + if (!ret) { + unpin_user_pages(mapping->pages, mapping->npages); + account_locked_vm(ghvm->mm, mapping->npages, false); + } kfree(mapping->pages); kfree(mapping->parcel.acl_entries); @@ -47,21 +63,32 @@ void gh_vm_mem_reclaim(struct gh_vm *ghvm, struct gh_vm_mem *mapping) list_del(&mapping->list); } +void gh_vm_mem_reclaim(struct gh_vm *ghvm) +{ + struct gh_vm_mem *mapping, *tmp; + + mutex_lock(&ghvm->mm_lock); + + list_for_each_entry_safe(mapping, tmp, &ghvm->memory_mappings, list) { + gh_vm_mem_reclaim_mapping(ghvm, mapping); + kfree(mapping); + } + + mutex_unlock(&ghvm->mm_lock); +} + struct gh_vm_mem *gh_vm_mem_find_by_addr(struct gh_vm *ghvm, u64 guest_phys_addr, u32 size) { - struct gh_vm_mem *mapping = NULL; - int ret; + struct gh_vm_mem *mapping; - ret = mutex_lock_interruptible(&ghvm->mm_lock); - if (ret) - return ERR_PTR(ret); + if (overflows_type(guest_phys_addr + size, u64)) + return NULL; + + mutex_lock(&ghvm->mm_lock); list_for_each_entry(mapping, &ghvm->memory_mappings, list) { - if (guest_phys_addr >= mapping->guest_phys_addr && - (guest_phys_addr + size <= mapping->guest_phys_addr + - (mapping->npages << PAGE_SHIFT))) { + if (gh_vm_mem_overlap(mapping, guest_phys_addr, size)) goto unlock; - } } mapping = NULL; @@ -70,91 +97,81 @@ unlock: return mapping; } -struct gh_vm_mem *gh_vm_mem_find_by_label(struct gh_vm *ghvm, u32 label) -{ - struct gh_vm_mem *mapping; - int ret; - - ret = mutex_lock_interruptible(&ghvm->mm_lock); - if (ret) - return ERR_PTR(ret); - - mapping = __gh_vm_mem_find_by_label(ghvm, label); - mutex_unlock(&ghvm->mm_lock); - - return mapping ? : ERR_PTR(-ENODEV); -} - int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *region, bool lend) { struct gh_vm_mem *mapping, *tmp_mapping; - struct gh_rm_mem_entry *mem_entries; - phys_addr_t curr_page, prev_page; + struct page *curr_page, *prev_page; struct gh_rm_mem_parcel *parcel; int i, j, pinned, ret = 0; + unsigned int gup_flags; size_t entry_size; u16 vmid; if (!region->memory_size || !PAGE_ALIGNED(region->memory_size) || - !PAGE_ALIGNED(region->userspace_addr) || !PAGE_ALIGNED(region->guest_phys_addr)) + !PAGE_ALIGNED(region->userspace_addr) || + !PAGE_ALIGNED(region->guest_phys_addr)) return -EINVAL; - if (region->guest_phys_addr + region->memory_size < region->guest_phys_addr) + if (overflows_type(region->guest_phys_addr + region->memory_size, u64)) return -EOVERFLOW; ret = mutex_lock_interruptible(&ghvm->mm_lock); if (ret) return ret; + /* Check label is unique */ mapping = __gh_vm_mem_find_by_label(ghvm, region->label); if (mapping) { - mutex_unlock(&ghvm->mm_lock); - return -EEXIST; + ret = -EEXIST; + goto unlock; } - mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); - if (!mapping) { - mutex_unlock(&ghvm->mm_lock); - return -ENOMEM; - } - - mapping->parcel.label = region->label; - mapping->guest_phys_addr = region->guest_phys_addr; - mapping->npages = region->memory_size >> PAGE_SHIFT; - parcel = &mapping->parcel; - parcel->mem_handle = GH_MEM_HANDLE_INVAL; /* to be filled later by mem_share/mem_lend */ - parcel->mem_type = GH_RM_MEM_TYPE_NORMAL; - /* Check for overlap */ list_for_each_entry(tmp_mapping, &ghvm->memory_mappings, list) { - if (!((mapping->guest_phys_addr + (mapping->npages << PAGE_SHIFT) <= - tmp_mapping->guest_phys_addr) || - (mapping->guest_phys_addr >= - tmp_mapping->guest_phys_addr + (tmp_mapping->npages << PAGE_SHIFT)))) { + if (gh_vm_mem_overlap(tmp_mapping, region->guest_phys_addr, + region->memory_size)) { ret = -EEXIST; - goto free_mapping; + goto unlock; } } - list_add(&mapping->list, &ghvm->memory_mappings); + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL_ACCOUNT); + if (!mapping) { + ret = -ENOMEM; + goto unlock; + } - mapping->pages = kcalloc(mapping->npages, sizeof(*mapping->pages), GFP_KERNEL); + mapping->guest_phys_addr = region->guest_phys_addr; + mapping->npages = region->memory_size >> PAGE_SHIFT; + parcel = &mapping->parcel; + parcel->label = region->label; + parcel->mem_handle = GH_MEM_HANDLE_INVAL; /* to be filled later by mem_share/mem_lend */ + parcel->mem_type = GH_RM_MEM_TYPE_NORMAL; + + ret = account_locked_vm(ghvm->mm, mapping->npages, true); + if (ret) + goto free_mapping; + + mapping->pages = kcalloc(mapping->npages, sizeof(*mapping->pages), GFP_KERNEL_ACCOUNT); if (!mapping->pages) { ret = -ENOMEM; mapping->npages = 0; /* update npages for reclaim */ - goto reclaim; + goto unlock_pages; } + gup_flags = FOLL_LONGTERM; + if (region->flags & GH_MEM_ALLOW_WRITE) + gup_flags |= FOLL_WRITE; + pinned = pin_user_pages_fast(region->userspace_addr, mapping->npages, - FOLL_WRITE | FOLL_LONGTERM, mapping->pages); + gup_flags, mapping->pages); if (pinned < 0) { ret = pinned; - mapping->npages = 0; /* update npages for reclaim */ - goto reclaim; + goto free_pages; } else if (pinned != mapping->npages) { ret = -EFAULT; mapping->npages = pinned; /* update npages for reclaim */ - goto reclaim; + goto unpin_pages; } if (lend) { @@ -164,15 +181,16 @@ int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *regio parcel->n_acl_entries = 2; mapping->share_type = VM_MEM_SHARE; } - parcel->acl_entries = kcalloc(parcel->n_acl_entries, sizeof(*parcel->acl_entries), - GFP_KERNEL); + parcel->acl_entries = kcalloc(parcel->n_acl_entries, + sizeof(*parcel->acl_entries), GFP_KERNEL); if (!parcel->acl_entries) { ret = -ENOMEM; - goto reclaim; + goto unpin_pages; } - parcel->acl_entries[0].vmid = cpu_to_le16(ghvm->vmid); - + /* acl_entries[0].vmid will be this VM's vmid. We'll fill it when the + * VM is starting and we know the VM's vmid. + */ if (region->flags & GH_MEM_ALLOW_READ) parcel->acl_entries[0].perms |= GH_RM_ACL_R; if (region->flags & GH_MEM_ALLOW_WRITE) @@ -180,78 +198,67 @@ int gh_vm_mem_alloc(struct gh_vm *ghvm, struct gh_userspace_memory_region *regio if (region->flags & GH_MEM_ALLOW_EXEC) parcel->acl_entries[0].perms |= GH_RM_ACL_X; - if (mapping->share_type == VM_MEM_SHARE) { + if (!lend) { ret = gh_rm_get_vmid(ghvm->rm, &vmid); if (ret) - goto reclaim; + goto free_acl; parcel->acl_entries[1].vmid = cpu_to_le16(vmid); /* Host assumed to have all these permissions. Gunyah will not - * grant new permissions if host actually had less than RWX - */ - parcel->acl_entries[1].perms |= GH_RM_ACL_R | GH_RM_ACL_W | GH_RM_ACL_X; + * grant new permissions if host actually had less than RWX + */ + parcel->acl_entries[1].perms = GH_RM_ACL_R | GH_RM_ACL_W | GH_RM_ACL_X; } - mem_entries = kcalloc(mapping->npages, sizeof(*mem_entries), GFP_KERNEL); - if (!mem_entries) { + parcel->n_mem_entries = 1; + for (i = 1; i < mapping->npages; i++) { + if (!pages_are_mergeable(mapping->pages[i - 1], mapping->pages[i])) + parcel->n_mem_entries++; + } + + parcel->mem_entries = kcalloc(parcel->n_mem_entries, + sizeof(parcel->mem_entries[0]), + GFP_KERNEL_ACCOUNT); + if (!parcel->mem_entries) { ret = -ENOMEM; - goto reclaim; + goto free_acl; } /* reduce number of entries by combining contiguous pages into single memory entry */ - prev_page = page_to_phys(mapping->pages[0]); - mem_entries[0].ipa_base = cpu_to_le64(prev_page); + prev_page = mapping->pages[0]; + parcel->mem_entries[0].ipa_base = cpu_to_le64(page_to_phys(prev_page)); entry_size = PAGE_SIZE; for (i = 1, j = 0; i < mapping->npages; i++) { - curr_page = page_to_phys(mapping->pages[i]); - if (curr_page - prev_page == PAGE_SIZE) { + curr_page = mapping->pages[i]; + if (pages_are_mergeable(prev_page, curr_page)) { entry_size += PAGE_SIZE; } else { - mem_entries[j].size = cpu_to_le64(entry_size); + parcel->mem_entries[j].size = cpu_to_le64(entry_size); j++; - mem_entries[j].ipa_base = cpu_to_le64(curr_page); + BUG_ON(j >= parcel->n_mem_entries); + parcel->mem_entries[j].ipa_base = + cpu_to_le64(page_to_phys(curr_page)); entry_size = PAGE_SIZE; } prev_page = curr_page; } - mem_entries[j].size = cpu_to_le64(entry_size); - - parcel->n_mem_entries = j + 1; - parcel->mem_entries = kmemdup(mem_entries, sizeof(*mem_entries) * parcel->n_mem_entries, - GFP_KERNEL); - kfree(mem_entries); - if (!parcel->mem_entries) { - ret = -ENOMEM; - goto reclaim; - } + parcel->mem_entries[j].size = cpu_to_le64(entry_size); + list_add(&mapping->list, &ghvm->memory_mappings); mutex_unlock(&ghvm->mm_lock); return 0; -reclaim: - gh_vm_mem_reclaim(ghvm, mapping); +free_acl: + kfree(parcel->acl_entries); +unpin_pages: + unpin_user_pages(mapping->pages, pinned); +free_pages: + kfree(mapping->pages); +unlock_pages: + account_locked_vm(ghvm->mm, mapping->npages, false); free_mapping: kfree(mapping); - mutex_unlock(&ghvm->mm_lock); - return ret; -} - -int gh_vm_mem_free(struct gh_vm *ghvm, u32 label) -{ - struct gh_vm_mem *mapping; - int ret; - - ret = mutex_lock_interruptible(&ghvm->mm_lock); - if (ret) - return ret; - - mapping = __gh_vm_mem_find_by_label(ghvm, label); - if (!mapping) - goto out; - - gh_vm_mem_reclaim(ghvm, mapping); - kfree(mapping); -out: +unlock: mutex_unlock(&ghvm->mm_lock); return ret; } diff --git a/include/linux/gunyah_rsc_mgr.h b/include/linux/gunyah_rsc_mgr.h index 7e2c9b7d5e20..27283c881ecb 100644 --- a/include/linux/gunyah_rsc_mgr.h +++ b/include/linux/gunyah_rsc_mgr.h @@ -81,16 +81,16 @@ enum gh_rm_mem_type { }; /* - * struct gh_rm_mem_parcel - Package info about memory to be lent/shared/donated/reclaimed + * struct gh_rm_mem_parcel - Info about memory to be lent/shared/donated/reclaimed * @mem_type: The type of memory: normal (DDR) or IO * @label: An client-specified identifier which can be used by the other VMs to identify the purpose * of the memory parcel. + * @n_acl_entries: Count of the number of entries in the @acl_entries array. * @acl_entries: An array of access control entries. Each entry specifies a VM and what access * is allowed for the memory parcel. - * @n_acl_entries: Count of the number of entries in the `acl_entries` array. - * @mem_entries: An list of regions to be associated with the memory parcel. Addresses should be + * @n_mem_entries: Count of the number of entries in the @mem_entries array. + * @mem_entries: An array of regions to be associated with the memory parcel. Addresses should be * (intermediate) physical addresses from Linux's perspective. - * @n_mem_entries: Count of the number of entries in the `mem_entries` array. * @mem_handle: On success, filled with memory handle that RM allocates for this memory parcel */ struct gh_rm_mem_parcel {