shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-06 10:58:48 +09:00
Code Issues Packages Projects Releases Wiki Activity

UPSTREAM: mm/mmap: free vm_area_struct without call_rcu in exit_mmap

Browse Source 
call_rcu() can take a long time when callback offloading is enabled.  Its
use in the vm_area_free can cause regressions in the exit path when
multiple VMAs are being freed.

Because exit_mmap() is called only after the last mm user drops its
refcount, the page fault handlers can't be racing with it.  Any other
possible user like oom-reaper or process_mrelease are already synchronized
using mmap_lock.  Therefore exit_mmap() can free VMAs directly, without
the use of call_rcu().

Expose __vm_area_free() and use it from exit_mmap() to avoid possible
call_rcu() floods and performance regressions caused by it.

Link: https://lkml.kernel.org/r/20230227173632.3292573-33-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

(cherry picked from commit 0d2ebf9c3f)

Bug: 161210518
Change-Id: I4fbf3ef38fdb22a3c80dcc61125ec21d2c426100
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
This commit is contained in:
Suren Baghdasaryan
2023-02-27 09:36:31 -08:00
committed by Carlos Llamas
parent e04309717e
commit 23fcd3167e
3 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/mm.h
View File

@@ -257,6 +257,8 @@ void setup_initial_init_mm(void *start_code, void *end_code,
struct vm_area_struct *vm_area_alloc(struct mm_struct *); struct vm_area_struct *vm_area_alloc(struct mm_struct *);
struct vm_area_struct *vm_area_dup(struct vm_area_struct *); struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
void vm_area_free(struct vm_area_struct *); void vm_area_free(struct vm_area_struct *);
/* Use only if VMA has no other users */
void __vm_area_free(struct vm_area_struct *vma);
#ifndef CONFIG_MMU #ifndef CONFIG_MMU
extern struct rb_root nommu_region_tree; extern struct rb_root nommu_region_tree;

2
kernel/fork.c
View File

@@ -487,7 +487,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
return new; return new;
} }
static void __vm_area_free(struct vm_area_struct *vma) void __vm_area_free(struct vm_area_struct *vma)
{ {
free_anon_vma_name(vma); free_anon_vma_name(vma);
kmem_cache_free(vm_area_cachep, vma); kmem_cache_free(vm_area_cachep, vma);

11
mm/mmap.c
View File

@@ -136,7 +136,7 @@ void unlink_file_vma(struct vm_area_struct *vma)
/* /*
* Close a vm structure and free it. * Close a vm structure and free it.
*/ */
static void remove_vma(struct vm_area_struct *vma) static void remove_vma(struct vm_area_struct *vma, bool unreachable)
{ {
might_sleep(); might_sleep();
if (vma->vm_ops && vma->vm_ops->close) if (vma->vm_ops && vma->vm_ops->close)
@@ -144,7 +144,10 @@ static void remove_vma(struct vm_area_struct *vma)
if (vma->vm_file) if (vma->vm_file)
fput(vma->vm_file); fput(vma->vm_file);
mpol_put(vma_policy(vma)); mpol_put(vma_policy(vma));
vm_area_free(vma); if (unreachable)
__vm_area_free(vma);
else
vm_area_free(vma);
} }
/* /*
@@ -2206,7 +2209,7 @@ static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
if (vma->vm_flags & VM_ACCOUNT) if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += nrpages; nr_accounted += nrpages;
vm_stat_account(mm, vma->vm_flags, -nrpages); vm_stat_account(mm, vma->vm_flags, -nrpages);
remove_vma(vma); remove_vma(vma, false);
} }
vm_unacct_memory(nr_accounted); vm_unacct_memory(nr_accounted);
validate_mm(mm); validate_mm(mm);
@@ -3179,7 +3182,7 @@ void exit_mmap(struct mm_struct *mm)
do { do {
if (vma->vm_flags & VM_ACCOUNT) if (vma->vm_flags & VM_ACCOUNT)
nr_accounted += vma_pages(vma); nr_accounted += vma_pages(vma);
remove_vma(vma); remove_vma(vma, true);
count++; count++;
cond_resched(); cond_resched();
} while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL);