diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 4bfed53e210e..9dd72b22e23b 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1226,6 +1226,10 @@ void do_user_addr_fault(struct pt_regs *regs, struct mm_struct *mm; vm_fault_t fault; unsigned int flags = FAULT_FLAG_DEFAULT; +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + struct vm_area_struct pvma; + unsigned long seq; +#endif tsk = current; mm = tsk->mm; @@ -1323,6 +1327,43 @@ void do_user_addr_fault(struct pt_regs *regs, } #endif +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + count_vm_event(SPF_ATTEMPT); + seq = mmap_seq_read_start(mm); + if (seq & 1) + goto spf_abort; + rcu_read_lock(); + vma = __find_vma(mm, address); + if (!vma || vma->vm_start > address) { + rcu_read_unlock(); + goto spf_abort; + } + pvma = *vma; + rcu_read_unlock(); + if (!mmap_seq_read_check(mm, seq)) + goto spf_abort; + vma = &pvma; + if (unlikely(access_error(error_code, vma))) + goto spf_abort; + fault = do_handle_mm_fault(vma, address, + flags | FAULT_FLAG_SPECULATIVE, seq, regs); + + if (!(fault & VM_FAULT_RETRY)) + goto done; + + /* Quick path to respond to signals */ + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + kernelmode_fixup_or_oops(regs, error_code, address, + SIGBUS, BUS_ADRERR, + ARCH_DEFAULT_PKEY); + return; + } + +spf_abort: + count_vm_event(SPF_ABORT); +#endif + /* * Kernel-mode access to the user address space should only occur * on well-defined single instructions listed in the exception @@ -1420,6 +1461,9 @@ good_area: } mmap_read_unlock(mm); +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT +done: +#endif if (likely(!(fault & VM_FAULT_ERROR))) return; diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index fb63d9dfee48..a9b9d19b9612 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -322,6 +322,11 @@ struct anon_vma_name { * per VM-area/task. A VM area is any part of the process virtual memory * space that has a special rule for the page-fault handlers (ie a shared * library, the executable area etc). + * + * Note that speculative page faults make an on-stack copy of the VMA, + * so the structure size matters. + * (TODO - it would be preferable to copy only the required vma attributes + * rather than the entire vma). */ struct vm_area_struct { /* The first cache line has the info for VMA tree walking. */ diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index a185cc75ff52..20d355fac63b 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -130,6 +130,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, #ifdef CONFIG_X86 DIRECT_MAP_LEVEL2_SPLIT, DIRECT_MAP_LEVEL3_SPLIT, +#endif +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + SPF_ATTEMPT, + SPF_ABORT, #endif NR_VM_EVENT_ITEMS }; diff --git a/mm/vmstat.c b/mm/vmstat.c index e24419ef6fc4..16d1e9426af5 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1379,6 +1379,10 @@ const char * const vmstat_text[] = { "direct_map_level2_splits", "direct_map_level3_splits", #endif +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + "spf_attempt", + "spf_abort", +#endif #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */ }; #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */