shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-06 02:50:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

FROMLIST: arm64/mm: attempt speculative mm faults first

Browse Source 
Attempt speculative mm fault handling first, and fall back to the
existing (non-speculative) code if that fails.

This follows the lines of the x86 speculative fault handling code,
but with some minor arch differences such as the way that the
VM_FAULT_BADACCESS case is handled.

Signed-off-by: Michel Lespinasse <michel@lespinasse.org>
Link: https://lore.kernel.org/all/20220128131006.67712-34-michel@lespinasse.org/
Bug: 161210518
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Change-Id: Iccd87036b15eebf2ff28fbb8022b07c9f91d7353
This commit is contained in:
Michel Lespinasse
2022-01-24 17:43:56 -08:00
committed by Todd Kjos
parent f03ec9d1c6
commit ac39e2e1eb
1 changed files with 62 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
arch/arm64/mm/fault.c
View File

@@ -25,6 +25,7 @@
#include <linux/perf_event.h>
#include <linux/preempt.h>
#include <linux/hugetlb.h>
#include <linux/vm_event_item.h>
#include <asm/acpi.h>
#include <asm/bug.h>
@@ -525,6 +526,11 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
unsigned long vm_flags;
unsigned int mm_flags = FAULT_FLAG_DEFAULT;
unsigned long addr = untagged_addr(far);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
struct vm_area_struct *vma;
struct vm_area_struct pvma;
unsigned long seq;
#endif
if (kprobe_page_fault(regs, esr))
return 0;
@@ -575,6 +581,59 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
/*
* No need to try speculative faults for kernel or
* single threaded user space.
*/
if (!(mm_flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)
goto no_spf;
count_vm_event(SPF_ATTEMPT);
seq = mmap_seq_read_start(mm);
if (seq & 1) {
count_vm_spf_event(SPF_ABORT_ODD);
goto spf_abort;
}
rcu_read_lock();
vma = __find_vma(mm, addr);
if (!vma || vma->vm_start > addr) {
rcu_read_unlock();
count_vm_spf_event(SPF_ABORT_UNMAPPED);
goto spf_abort;
}
if (!vma_is_anonymous(vma)) {
rcu_read_unlock();
count_vm_spf_event(SPF_ABORT_NO_SPECULATE);
goto spf_abort;
}
pvma = *vma;
rcu_read_unlock();
if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))
goto spf_abort;
vma = &pvma;
if (!(vma->vm_flags & vm_flags)) {
count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);
goto spf_abort;
}
fault = do_handle_mm_fault(vma, addr & PAGE_MASK,
mm_flags | FAULT_FLAG_SPECULATIVE, seq, regs);
/* Quick path to respond to signals */
if (fault_signal_pending(fault, regs)) {
if (!user_mode(regs))
goto no_context;
return 0;
}
if (!(fault & VM_FAULT_RETRY))
goto done;
spf_abort:
count_vm_event(SPF_ABORT);
no_spf:
#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
/*
* As per x86, we may deadlock here. However, since the kernel only
* validly references user space from well defined areas of the code,
@@ -615,6 +674,9 @@ retry:
}
}
mmap_read_unlock(mm);
#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
done:
#endif
/*
* Handle the "normal" (no error) case first.