From 07622f00e6fd6fc1bde008e2de62d6486a46ae39 Mon Sep 17 00:00:00 2001 From: Jianxiong Pan Date: Thu, 22 Sep 2022 17:25:35 +0800 Subject: [PATCH] mm: pin locked file after fault. [1/2] PD#SWPL-95400 Problem: If we pin a file in android, the whole data of this file will be loaded to DDR, but most of these data may not be used, this caused memory waste. Solution: delay mark mlocked flags for unevictable pages until it has been faulted. You can use following command to disable this function: echo 0 > /proc/sys/vm/shrink_unevictable Verify: sc2_ah212 Change-Id: I05bb7654cc228604fe692429efbe40af713ca5c3 Signed-off-by: Jianxiong Pan --- mm/gup.c | 19 +++++++++++++ mm/memory.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++- mm/migrate.c | 7 +++++ mm/mlock.c | 10 +++++++ mm/rmap.c | 17 +++++++++++- mm/vmscan.c | 10 +++++++ 6 files changed, 139 insertions(+), 2 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index f8263f945838..4ba357c8a0f4 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -21,6 +21,9 @@ #include #include +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +#include +#endif #include "internal.h" @@ -624,6 +627,21 @@ retry: * when it attempts to reclaim the page. */ if (page->mapping && trylock_page(page)) { + #ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + struct address_space *mapping; + + mapping = page_mapping(page); + if (mapping && + test_bit(AS_LOCK_MAPPING, &mapping->flags) && + sysctrl_shrink_unevictable) { + unlock_page(page); + } else { + /* same as #else */ + lru_add_drain(); + mlock_vma_page(page); + unlock_page(page); + } + #else lru_add_drain(); /* push cached pages to LRU */ /* * Because we lock page here, and migration is @@ -633,6 +651,7 @@ retry: */ mlock_vma_page(page); unlock_page(page); + #endif } } out: diff --git a/mm/memory.c b/mm/memory.c index 0b14e86ee68c..e42b3473f6d7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -85,6 +85,12 @@ #include #include #include +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +#include +#endif +#ifdef CONFIG_AMLOGIC_MEM_DEBUG +#include +#endif #include "pgalloc-track.h" #include "internal.h" @@ -4929,6 +4935,48 @@ unlock: return 0; } +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +static bool __restore_locked_page(struct page *page, struct vm_area_struct *vma, + unsigned long addr, void *arg) +{ + int ret; + pte_t old_pte, *pte; + spinlock_t *ptl; /* pte lock */ + + if (vma->vm_flags & (VM_LOCKED | VM_LOCKONFAULT)) { + if (addr < vma->vm_start || addr >= vma->vm_end) + return -EFAULT; + if (!page_count(page)) + return -EINVAL; + + pte = get_locked_pte(vma->vm_mm, addr, &ptl); + if (!pte) + return true; + old_pte = *pte; + pte_unmap_unlock(pte, ptl); + /* already refaulted */ + if (pte_valid(old_pte) && pte_pfn(old_pte) == page_to_pfn(page)) + return true; + + ret = insert_page(vma, addr, page, vma->vm_page_prot); + pr_debug("%s, restore page:%lx for addr:%lx, vma:%px, ret:%d, old_pte:%llx\n", + __func__, page_to_pfn(page), addr, vma, ret, + (unsigned long long)pte_val(old_pte)); + return ret ? false : true; + } + return true; /* keep loop */ +} + +static void restore_locked_page(struct page *page) +{ + struct rmap_walk_control rwc = { + .rmap_one = __restore_locked_page, + }; + + rmap_walk(page, &rwc); +} +#endif + /* * By the time we get here, we already hold the mm semaphore * @@ -4944,6 +4992,9 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, .flags = flags, .pgoff = linear_page_index(vma, address), .gfp_mask = __get_fault_gfp_mask(vma), + #ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + .pte = NULL, + #endif }; unsigned int dirty = flags & FAULT_FLAG_WRITE; struct mm_struct *mm = vma->vm_mm; @@ -5055,7 +5106,14 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, if (!vmf.pte && uffd_missing_sigbus) return VM_FAULT_SIGBUS; - return handle_pte_fault(&vmf); +#ifdef CONFIG_AMLOGIC_MEM_DEBUG + ret = handle_pte_fault(&vmf); + if (!ret && vma->vm_flags & VM_LOCKED) + mlock_fault_size++; + return ret; +#else + return handle_pte_fault(&vmf); +#endif spf_fail: speculative_page_walk_end(); @@ -5185,7 +5243,25 @@ retry_pud: } } +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + ret = handle_pte_fault(&vmf); + if (!ret) { + struct page *page = NULL; + + page = aml_mlock_page_as_lock_mapping(vma, mm, &vmf, address); + if (page) + restore_locked_page(page); + } + return ret; +#endif +#ifdef CONFIG_AMLOGIC_MEM_DEBUG + ret = handle_pte_fault(&vmf); + if (!ret && vma->vm_flags & VM_LOCKED) + mlock_fault_size++; + return ret; +#else return handle_pte_fault(&vmf); +#endif } /** diff --git a/mm/migrate.c b/mm/migrate.c index c89a3271fa9a..9b14fe8603bb 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -54,6 +54,9 @@ #include #include #endif +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +#include +#endif #include @@ -1295,6 +1298,10 @@ out: else put_page(newpage); } +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + if (reason == MR_CONTIG_RANGE && rc == MIGRATEPAGE_SUCCESS) + aml_clear_pin_locked_file(page); +#endif return rc; } diff --git a/mm/mlock.c b/mm/mlock.c index eec2418f3336..96309eeed860 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -25,6 +25,9 @@ #include #include #include +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +#include +#endif #include "internal.h" @@ -193,6 +196,7 @@ unsigned int munlock_vma_page(struct page *page) /* * convert get_user_pages() return value to posix mlock() error */ +#ifndef CONFIG_AMLOGIC_PIN_LOCKED_FILE_V2 static int __mlock_posix_error_return(long retval) { if (retval == -EFAULT) @@ -201,6 +205,7 @@ static int __mlock_posix_error_return(long retval) retval = -EAGAIN; return retval; } +#endif /* * Prepare page for fast batched LRU putback via putback_lru_evictable_pagevec() @@ -588,6 +593,9 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, tmp = vma->vm_end; if (tmp > end) tmp = end; + #ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + reset_page_vma_flags(vma, flags); + #endif error = mlock_fixup(vma, &prev, nstart, tmp, newflags); if (error) break; @@ -686,9 +694,11 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if (error) return error; +#ifndef CONFIG_AMLOGIC_PIN_LOCKED_FILE_V2 error = __mm_populate(start, len, 0); if (error) return __mlock_posix_error_return(error); +#endif return 0; } diff --git a/mm/rmap.c b/mm/rmap.c index 761b7f26adc9..e2e606453f06 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -80,6 +80,9 @@ #include #include +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +#include +#endif #include "internal.h" @@ -1471,8 +1474,14 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, * cleared). But stop unmapping even in those cases. */ if (!PageTransCompound(page) || (PageHead(page) && - !PageDoubleMap(page) && !PageAnon(page))) + !PageDoubleMap(page) && !PageAnon(page))) + #ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + /* only keep refault pages */ + if (aml_is_pin_locked_file(page)) + mlock_vma_page(page); + #else mlock_vma_page(page); + #endif page_vma_mapped_walk_done(&pvmw); ret = false; break; @@ -2036,7 +2045,13 @@ static bool page_mlock_one(struct page *page, struct vm_area_struct *vma, * nor on an Anon THP (which may still be PTE-mapped * after DoubleMap was cleared). */ + #ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + /* only keep refault pages */ + if (aml_is_pin_locked_file(page)) + mlock_vma_page(page); + #else mlock_vma_page(page); + #endif /* * No need to scan further once the page is marked * as mlocked. diff --git a/mm/vmscan.c b/mm/vmscan.c index 635bd6abb625..37e503d1bcf1 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -77,6 +77,9 @@ #ifdef CONFIG_AMLOGIC_CMA #include #endif +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE +#include +#endif EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end); @@ -1674,6 +1677,13 @@ retry: enum ttu_flags flags = TTU_BATCH_FLUSH; bool was_swapbacked = PageSwapBacked(page); +#ifdef CONFIG_AMLOGIC_PIN_LOCKED_FILE + if (mapping && + test_bit(AS_LOCK_MAPPING, &mapping->flags) && + !aml_is_pin_locked_file(page) && + !PageMlocked(page)) + flags |= TTU_IGNORE_MLOCK; +#endif if (unlikely(PageTransHuge(page))) flags |= TTU_SPLIT_HUGE_PMD;