From a17e132ec4f290621666311e73f43202706d2743 Mon Sep 17 00:00:00 2001 From: Minchan Kim Date: Thu, 27 Oct 2022 08:29:17 -0700 Subject: [PATCH] ANDROID: vendor hook to control blk_plug for memory reclaim Add vendor hook to contorl blk plugging. Bug: 255471591 Bug: 238728493 Change-Id: I96b73cec14f0d2fea46a4828526e6ae5aa5c71b7 Signed-off-by: Minchan Kim --- drivers/android/vendor_hooks.c | 3 +++ include/trace/hooks/mm.h | 9 +++++++++ mm/madvise.c | 9 +++++++-- mm/vmscan.c | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 442537ee4fc1..bfd9a74a0f61 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -283,6 +283,9 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_get_from_fragment_pool); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exclude_reserved_zone); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_include_reserved_zone); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_slowpath); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_madvise_blk_plug); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_inactive_list_blk_plug); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_reclaim_pages_plug); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zap_pte_range_tlb_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zap_pte_range_tlb_force_flush); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_zap_pte_range_tlb_end); diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index 94d18851fc2a..236aaeeff274 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -100,6 +100,15 @@ DECLARE_HOOK(android_vh_alloc_pages_slowpath, DECLARE_HOOK(android_vh_cma_alloc_adjust, TP_PROTO(struct zone *zone, bool *is_cma_alloc), TP_ARGS(zone, is_cma_alloc)); +DECLARE_HOOK(android_vh_do_madvise_blk_plug, + TP_PROTO(int behavior, bool *do_plug), + TP_ARGS(behavior, do_plug)); +DECLARE_HOOK(android_vh_shrink_inactive_list_blk_plug, + TP_PROTO(bool *do_plug), + TP_ARGS(do_plug)); +DECLARE_HOOK(android_vh_reclaim_pages_plug, + TP_PROTO(bool *do_plug), + TP_ARGS(do_plug)); DECLARE_HOOK(android_vh_zap_pte_range_tlb_start, TP_PROTO(void *unused), TP_ARGS(unused)); diff --git a/mm/madvise.c b/mm/madvise.c index 8920a7125389..937301fb9bc5 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -31,6 +31,7 @@ #include #include #include +#include #include @@ -1266,6 +1267,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh int write; size_t len; struct blk_plug plug; + bool do_plug = true; start = untagged_addr(start); @@ -1300,10 +1302,13 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh mmap_read_lock(mm); } - blk_start_plug(&plug); + trace_android_vh_do_madvise_blk_plug(behavior, &do_plug); + if (do_plug) + blk_start_plug(&plug); error = madvise_walk_vmas(mm, start, end, behavior, madvise_vma_behavior); - blk_finish_plug(&plug); + if (do_plug) + blk_finish_plug(&plug); if (write) mmap_write_unlock(mm); else diff --git a/mm/vmscan.c b/mm/vmscan.c index d09d63861544..f4a22e18951d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -70,6 +70,9 @@ #undef CREATE_TRACE_POINTS #include +#undef CREATE_TRACE_POINTS +#include + EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end); @@ -2005,6 +2008,8 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, enum vm_event_item item; struct pglist_data *pgdat = lruvec_pgdat(lruvec); bool stalled = false; + struct blk_plug plug; + bool do_plug = false; while (unlikely(too_many_isolated(pgdat, file, sc))) { if (stalled) @@ -2038,11 +2043,16 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, if (nr_taken == 0) return 0; + trace_android_vh_shrink_inactive_list_blk_plug(&do_plug); + if (do_plug) + blk_start_plug(&plug); nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false); spin_lock_irq(&pgdat->lru_lock); move_pages_to_lru(lruvec, &page_list); + if (do_plug) + blk_finish_plug(&plug); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); lru_note_cost(lruvec, file, stat.nr_pageout); @@ -2188,6 +2198,8 @@ unsigned long reclaim_pages(struct list_head *page_list) LIST_HEAD(node_page_list); struct reclaim_stat dummy_stat; struct page *page; + struct blk_plug plug; + bool do_plug = false; struct scan_control sc = { .gfp_mask = GFP_KERNEL, .priority = DEF_PRIORITY, @@ -2196,6 +2208,10 @@ unsigned long reclaim_pages(struct list_head *page_list) .may_swap = 1, }; + trace_android_vh_reclaim_pages_plug(&do_plug); + if (do_plug) + blk_start_plug(&plug); + while (!list_empty(page_list)) { page = lru_to_page(page_list); if (nid == NUMA_NO_NODE) { @@ -2231,6 +2247,8 @@ unsigned long reclaim_pages(struct list_head *page_list) putback_lru_page(page); } } + if (do_plug) + blk_finish_plug(&plug); return nr_reclaimed; }