From f60c5572d20ac22d7d5a7f86c3aace1ffb857962 Mon Sep 17 00:00:00 2001 From: Heesub Shin Date: Sun, 6 Jan 2013 18:10:00 -0800 Subject: [PATCH] ANDROID: cma: redirect page allocation to CMA CMA pages are designed to be used as fallback for movable allocations and cannot be used for non-movable allocations. If CMA pages are utilized poorly, non-movable allocations may end up getting starved if all regular movable pages are allocated and the only pages left are CMA. Always using CMA pages first creates unacceptable performance problems. As a midway alternative, use CMA pages for certain userspace allocations. The userspace pages can be migrated or dropped quickly which giving decent utilization. Additionally, add a fall-backs for failed CMA allocations in rmqueue() and __rmqueue_pcplist() (the latter addition being driven by a report by the kernel test robot); these fallbacks were dealt with differently in the original version of the patch as the rmqueue() call chain has changed). Bug: 158645321 Link: https://lore.kernel.org/lkml/cover.1604282969.git.cgoldswo@codeaurora.org/ Change-Id: Iad46f0405b416e29ae788f82b79c9953513a9c9d Reported-by: kernel test robot Signed-off-by: Kyungmin Park Signed-off-by: Heesub Shin Signed-off-by: Vinayak Menon [cgoldswo@codeaurora.org: Place in bugfixes; remove cma_alloc zone flag] Signed-off-by: Chris Goldsworthy [isaacm@codeaurora.org: Resolve merge conflicts to account for new mm features] Signed-off-by: Isaac J. Manjarres [quic_sukadev@quicinc.com: dropped unused gfp_flags parameter to __rmqueue_pcplist(), resolved some conflicts] Signed-off-by: Sukadev Bhattiprolu --- arch/arm64/mm/fault.c | 3 +- arch/x86/include/asm/page.h | 2 +- include/linux/gfp_types.h | 18 +++++++-- include/linux/highmem.h | 2 +- include/linux/mmzone.h | 2 + mm/page_alloc.c | 75 +++++++++++++++++++++++-------------- 6 files changed, 67 insertions(+), 35 deletions(-) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 3ebdd643de23..2d8d121c0b10 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -956,7 +957,7 @@ NOKPROBE_SYMBOL(do_debug_exception); struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { - gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; + gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO | __GFP_CMA; /* * If the page is mapped with PROT_MTE, initialise the tags at the diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h index 9cc82f305f4b..179b36063d27 100644 --- a/arch/x86/include/asm/page.h +++ b/arch/x86/include/asm/page.h @@ -35,7 +35,7 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, } #define alloc_zeroed_user_highpage_movable(vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO | __GFP_CMA, vma, vaddr) #define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE #ifndef __pa diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index d88c46ca82e1..872d1d5f0de3 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -55,10 +55,19 @@ typedef unsigned int __bitwise gfp_t; #define ___GFP_SKIP_KASAN_UNPOISON 0 #define ___GFP_SKIP_KASAN_POISON 0 #endif -#ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x8000000u +#ifdef CONFIG_CMA +#define ___GFP_CMA 0x8000000u #else -#define ___GFP_NOLOCKDEP 0 +#define ___GFP_CMA 0 +#endif +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_CMA +#define ___GFP_NOLOCKDEP 0x10000000u +#else +#define ___GFP_NOLOCKDEP 0x8000000u +#endif +#else +#define ___GFP_NOLOCKDEP 0 #endif /* If the above are modified, __GFP_BITS_SHIFT may need updating */ @@ -73,6 +82,7 @@ typedef unsigned int __bitwise gfp_t; #define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) #define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define __GFP_CMA ((__force gfp_t)___GFP_CMA) #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) /** @@ -256,7 +266,7 @@ typedef unsigned int __bitwise gfp_t; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP) + IS_ENABLED(CONFIG_CMA)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** diff --git a/include/linux/highmem.h b/include/linux/highmem.h index e9912da5441b..94b50dc0e131 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -227,7 +227,7 @@ static inline struct page * alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { - struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); + struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_CMA, vma, vaddr); if (page) clear_user_highpage(page, vaddr); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a0a06bc39245..60439e1d5aeb 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -73,9 +73,11 @@ extern const char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define get_cma_migrate_type() MIGRATE_CMA #else # define is_migrate_cma(migratetype) false # define is_migrate_cma_page(_page) false +# define get_cma_migrate_type() MIGRATE_MOVABLE #endif static inline bool is_migrate_movable(int mt) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 430438409adb..619ec627ed33 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3124,33 +3124,38 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, { struct page *page; - if (IS_ENABLED(CONFIG_CMA)) { - /* - * Balance movable allocations between regular and CMA areas by - * allocating from CMA when over half of the zone's free memory - * is in the CMA area. - */ - if (alloc_flags & ALLOC_CMA && - zone_page_state(zone, NR_FREE_CMA_PAGES) > - zone_page_state(zone, NR_FREE_PAGES) / 2) { - page = __rmqueue_cma_fallback(zone, order); - if (page) - return page; - } - } retry: page = __rmqueue_smallest(zone, order, migratetype); - if (unlikely(!page)) { - if (alloc_flags & ALLOC_CMA) - page = __rmqueue_cma_fallback(zone, order); - if (!page && __rmqueue_fallback(zone, order, migratetype, - alloc_flags)) - goto retry; - } + if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype, + alloc_flags)) + goto retry; + return page; } +#ifdef CONFIG_CMA +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + struct page *page = __rmqueue_cma_fallback(zone, order); + + if (page) + trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA, + pcp_allowed_order(order) && + migratetype < MIGRATE_PCPTYPES); + return page; +} +#else +static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + return NULL; +} +#endif + /* * Obtain a specified number of elements from the buddy allocator, all under * a single hold of the lock, for efficiency. Add them to the supplied list. @@ -3165,8 +3170,14 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, /* Caller must hold IRQ-safe pcp->lock so IRQs are disabled. */ spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order, migratetype, - alloc_flags); + struct page *page; + + if (is_migrate_cma(migratetype)) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, alloc_flags); + if (unlikely(page == NULL)) break; @@ -3748,7 +3759,12 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, if (order > 0 && alloc_flags & ALLOC_HARDER) page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); if (!page) { - page = __rmqueue(zone, order, migratetype, alloc_flags); + if (alloc_flags & ALLOC_CMA && migratetype == MIGRATE_MOVABLE) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, + alloc_flags); if (!page) { spin_unlock_irqrestore(&zone->lock, flags); return NULL; @@ -3789,9 +3805,12 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, */ if (batch > 1) batch = max(batch >> order, 2); - alloced = rmqueue_bulk(zone, order, - batch, list, - migratetype, alloc_flags); + if (migratetype == MIGRATE_MOVABLE && alloc_flags & ALLOC_CMA) + alloced = rmqueue_bulk(zone, order, batch, list, + get_cma_migrate_type(), alloc_flags); + if (unlikely(list_empty(list))) + alloced = rmqueue_bulk(zone, order, batch, list, migratetype, + alloc_flags); pcp->count += alloced << order; if (unlikely(list_empty(list))) @@ -4195,7 +4214,7 @@ static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, unsigned int alloc_flags) { #ifdef CONFIG_CMA - if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) + if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && gfp_mask & __GFP_CMA) alloc_flags |= ALLOC_CMA; #endif return alloc_flags;