diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 20a40e94757f..412d22b13a7c 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -442,3 +442,8 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_folio_trylock_clear); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_folio_trylock_set); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_trylock_failed_folio); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_hibernate_state); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swapmem_gather_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swapmem_gather_add_bypass); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_swapmem_gather_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_oom_swapmem_gather_init); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_oom_swapmem_gather_finish); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index b8527123813f..194f5536c149 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -246,6 +246,21 @@ DECLARE_HOOK(android_vh_shmem_swapin_folio, DECLARE_HOOK(android_vh_madvise_cold_or_pageout_page, TP_PROTO(bool pageout, struct page *page), TP_ARGS(pageout, page)); +DECLARE_HOOK(android_vh_swapmem_gather_init, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_swapmem_gather_add_bypass, + TP_PROTO(struct mm_struct *mm, swp_entry_t entry, bool *bypass), + TP_ARGS(mm, entry, bypass)); +DECLARE_HOOK(android_vh_swapmem_gather_finish, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_oom_swapmem_gather_init, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); +DECLARE_HOOK(android_vh_oom_swapmem_gather_finish, + TP_PROTO(struct mm_struct *mm), + TP_ARGS(mm)); #endif /* _TRACE_HOOK_MM_H */ /* This part must be outside protection */ diff --git a/mm/memory.c b/mm/memory.c index 3ee3360706ac..4c54877c04a0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1426,6 +1426,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pte_t *start_pte; pte_t *pte; swp_entry_t entry; + bool bypass = false; tlb_change_page_size(tlb, PAGE_SIZE); again: @@ -1498,6 +1499,9 @@ again: if (!should_zap_cows(details)) continue; rss[MM_SWAPENTS]--; + trace_android_vh_swapmem_gather_add_bypass(mm, entry, &bypass); + if (bypass) + goto skip; if (unlikely(!free_swap_and_cache(entry))) print_bad_pte(vma, addr, ptent, NULL); } else if (is_migration_entry(entry)) { @@ -1517,6 +1521,7 @@ again: /* We should have covered all the swap entry types */ WARN_ON_ONCE(1); } +skip: pte_clear_not_present_full(mm, addr, pte, tlb->fullmm); zap_install_uffd_wp_if_needed(vma, addr, pte, details, ptent); } while (pte++, addr += PAGE_SIZE, addr != end); diff --git a/mm/mmap.c b/mm/mmap.c index efe4859cd9e6..f38b3b610c72 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -3329,9 +3329,11 @@ void exit_mmap(struct mm_struct *mm) lru_add_drain(); flush_cache_mm(mm); tlb_gather_mmu_fullmm(&tlb, mm); + trace_android_vh_swapmem_gather_init(mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX, vma->vm_end, ULONG_MAX, false); + trace_android_vh_swapmem_gather_finish(mm); mmap_read_unlock(mm); /* diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 7a9e1c7f636c..18f57ec75233 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -53,6 +53,9 @@ #define CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include + static int sysctl_panic_on_oom; static int sysctl_oom_kill_allocating_task; static int sysctl_oom_dump_tasks = 1; @@ -525,6 +528,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) */ set_bit(MMF_UNSTABLE, &mm->flags); + trace_android_vh_oom_swapmem_gather_init(mm); for_each_vma(vmi, vma) { if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) continue; @@ -557,6 +561,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) tlb_finish_mmu(&tlb); } } + trace_android_vh_oom_swapmem_gather_finish(mm); return ret; }