diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 3ebdd643de23..2d8d121c0b10 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -956,7 +957,7 @@ NOKPROBE_SYMBOL(do_debug_exception); struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { - gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; + gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO | __GFP_CMA; /* * If the page is mapped with PROT_MTE, initialise the tags at the diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h index 9cc82f305f4b..179b36063d27 100644 --- a/arch/x86/include/asm/page.h +++ b/arch/x86/include/asm/page.h @@ -35,7 +35,7 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, } #define alloc_zeroed_user_highpage_movable(vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO | __GFP_CMA, vma, vaddr) #define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE #ifndef __pa diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index d88c46ca82e1..872d1d5f0de3 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -55,10 +55,19 @@ typedef unsigned int __bitwise gfp_t; #define ___GFP_SKIP_KASAN_UNPOISON 0 #define ___GFP_SKIP_KASAN_POISON 0 #endif -#ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x8000000u +#ifdef CONFIG_CMA +#define ___GFP_CMA 0x8000000u #else -#define ___GFP_NOLOCKDEP 0 +#define ___GFP_CMA 0 +#endif +#ifdef CONFIG_LOCKDEP +#ifdef CONFIG_CMA +#define ___GFP_NOLOCKDEP 0x10000000u +#else +#define ___GFP_NOLOCKDEP 0x8000000u +#endif +#else +#define ___GFP_NOLOCKDEP 0 #endif /* If the above are modified, __GFP_BITS_SHIFT may need updating */ @@ -73,6 +82,7 @@ typedef unsigned int __bitwise gfp_t; #define __GFP_HIGHMEM ((__force gfp_t)___GFP_HIGHMEM) #define __GFP_DMA32 ((__force gfp_t)___GFP_DMA32) #define __GFP_MOVABLE ((__force gfp_t)___GFP_MOVABLE) /* ZONE_MOVABLE allowed */ +#define __GFP_CMA ((__force gfp_t)___GFP_CMA) #define GFP_ZONEMASK (__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE) /** @@ -256,7 +266,7 @@ typedef unsigned int __bitwise gfp_t; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP) + IS_ENABLED(CONFIG_CMA)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** diff --git a/include/linux/highmem.h b/include/linux/highmem.h index e9912da5441b..94b50dc0e131 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -227,7 +227,7 @@ static inline struct page * alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { - struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); + struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_CMA, vma, vaddr); if (page) clear_user_highpage(page, vaddr); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a0a06bc39245..60439e1d5aeb 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -73,9 +73,11 @@ extern const char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define get_cma_migrate_type() MIGRATE_CMA #else # define is_migrate_cma(migratetype) false # define is_migrate_cma_page(_page) false +# define get_cma_migrate_type() MIGRATE_MOVABLE #endif static inline bool is_migrate_movable(int mt) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 430438409adb..619ec627ed33 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3124,33 +3124,38 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, { struct page *page; - if (IS_ENABLED(CONFIG_CMA)) { - /* - * Balance movable allocations between regular and CMA areas by - * allocating from CMA when over half of the zone's free memory - * is in the CMA area. - */ - if (alloc_flags & ALLOC_CMA && - zone_page_state(zone, NR_FREE_CMA_PAGES) > - zone_page_state(zone, NR_FREE_PAGES) / 2) { - page = __rmqueue_cma_fallback(zone, order); - if (page) - return page; - } - } retry: page = __rmqueue_smallest(zone, order, migratetype); - if (unlikely(!page)) { - if (alloc_flags & ALLOC_CMA) - page = __rmqueue_cma_fallback(zone, order); - if (!page && __rmqueue_fallback(zone, order, migratetype, - alloc_flags)) - goto retry; - } + if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype, + alloc_flags)) + goto retry; + return page; } +#ifdef CONFIG_CMA +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + struct page *page = __rmqueue_cma_fallback(zone, order); + + if (page) + trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA, + pcp_allowed_order(order) && + migratetype < MIGRATE_PCPTYPES); + return page; +} +#else +static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype, + unsigned int alloc_flags) +{ + return NULL; +} +#endif + /* * Obtain a specified number of elements from the buddy allocator, all under * a single hold of the lock, for efficiency. Add them to the supplied list. @@ -3165,8 +3170,14 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, /* Caller must hold IRQ-safe pcp->lock so IRQs are disabled. */ spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order, migratetype, - alloc_flags); + struct page *page; + + if (is_migrate_cma(migratetype)) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, alloc_flags); + if (unlikely(page == NULL)) break; @@ -3748,7 +3759,12 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, if (order > 0 && alloc_flags & ALLOC_HARDER) page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); if (!page) { - page = __rmqueue(zone, order, migratetype, alloc_flags); + if (alloc_flags & ALLOC_CMA && migratetype == MIGRATE_MOVABLE) + page = __rmqueue_cma(zone, order, migratetype, + alloc_flags); + else + page = __rmqueue(zone, order, migratetype, + alloc_flags); if (!page) { spin_unlock_irqrestore(&zone->lock, flags); return NULL; @@ -3789,9 +3805,12 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, */ if (batch > 1) batch = max(batch >> order, 2); - alloced = rmqueue_bulk(zone, order, - batch, list, - migratetype, alloc_flags); + if (migratetype == MIGRATE_MOVABLE && alloc_flags & ALLOC_CMA) + alloced = rmqueue_bulk(zone, order, batch, list, + get_cma_migrate_type(), alloc_flags); + if (unlikely(list_empty(list))) + alloced = rmqueue_bulk(zone, order, batch, list, migratetype, + alloc_flags); pcp->count += alloced << order; if (unlikely(list_empty(list))) @@ -4195,7 +4214,7 @@ static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, unsigned int alloc_flags) { #ifdef CONFIG_CMA - if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) + if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && gfp_mask & __GFP_CMA) alloc_flags |= ALLOC_CMA; #endif return alloc_flags;