diff --git a/mm/page_alloc.c b/mm/page_alloc.c index f33cb1cbe677..28004c549031 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -211,6 +211,38 @@ EXPORT_PER_CPU_SYMBOL(numa_node); DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key); +/* + * By default, restrict_cma_redirect is set to true, so only MOVABLE allocations + * marked __GFP_CMA are eligible to be redirected to CMA region. These allocations + * are redirected if *any* free space is available in the CMA region. + * When restrict_cma_redirect is false, all movable allocations + * are eligible for redirection to CMA region (i.e movable allocations are + * not restricted from CMA region), when there is sufficient space there. + * (see __rmqueue()). + * + */ +DEFINE_STATIC_KEY_TRUE(restrict_cma_redirect); + +static int __init restrict_cma_redirect_setup(char *str) +{ +#ifdef CONFIG_CMA + bool res; + int ret; + ret = kstrtobool(str, &res); + if (!ret && res == false) + static_branch_disable(&restrict_cma_redirect); +#else + pr_warn("CONFIG_CMA not set. Ignoring restrict_cma_redirect option\n"); +#endif + return 1; +} +__setup("restrict_cma_redirect=", restrict_cma_redirect_setup); + +static inline bool cma_redirect_restricted(void) +{ + return static_branch_likely(&restrict_cma_redirect); +} + #ifdef CONFIG_HAVE_MEMORYLESS_NODES /* * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. @@ -3177,6 +3209,21 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, if (page) return page; + if (IS_ENABLED(CONFIG_CMA)) { + /* + * Balance movable allocations between regular and CMA areas by + * allocating from CMA when over half of the zone's free memory + * is in the CMA area. + */ + if (!cma_redirect_restricted() && alloc_flags & ALLOC_CMA && + zone_page_state(zone, NR_FREE_CMA_PAGES) > + zone_page_state(zone, NR_FREE_PAGES) / 2) { + page = __rmqueue_cma_fallback(zone, order); + if (page) + return page; + } + } + retry: page = __rmqueue_smallest(zone, order, migratetype); @@ -3186,6 +3233,9 @@ retry: if (unlikely(!page) && (migratetype == MIGRATE_MOVABLE)) trace_android_vh_rmqueue_cma_fallback(zone, order, &page); + if (!cma_redirect_restricted() && !page && alloc_flags & ALLOC_CMA) + page = __rmqueue_cma_fallback(zone, order); + if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype, alloc_flags)) goto retry; @@ -3231,7 +3281,12 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, for (i = 0; i < count; ++i) { struct page *page; - if (is_migrate_cma(migratetype)) + /* + * If CMA redirect is restricted, use CMA region only for + * MIGRATE_CMA pages. cma_rediret_restricted() is false + * if CONFIG_CMA is not set. + */ + if (cma_redirect_restricted() && is_migrate_cma(migratetype)) page = __rmqueue_cma(zone, order, migratetype, alloc_flags); else @@ -3880,7 +3935,8 @@ struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone, if (alloc_flags & ALLOC_HIGHATOMIC) page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); if (!page) { - if (alloc_flags & ALLOC_CMA && migratetype == MIGRATE_MOVABLE) + if (cma_redirect_restricted() && alloc_flags & ALLOC_CMA && + migratetype == MIGRATE_MOVABLE) page = __rmqueue_cma(zone, order, migratetype, alloc_flags); else @@ -3924,7 +3980,8 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, do { /* First try to get CMA pages */ - if (migratetype == MIGRATE_MOVABLE && alloc_flags & ALLOC_CMA) + if (cma_redirect_restricted() && migratetype == MIGRATE_MOVABLE && + alloc_flags & ALLOC_CMA) list = get_populated_pcp_list(zone, order, pcp, get_cma_migrate_type(), alloc_flags); if (list == NULL) { @@ -4344,7 +4401,12 @@ static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask, unsigned int alloc_flags) { #ifdef CONFIG_CMA - if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && gfp_mask & __GFP_CMA) + /* + * If cma_redirect_restricted is true, set ALLOC_CMA only for + * movable allocations that have __GFP_CMA. + */ + if ((!cma_redirect_restricted() || gfp_mask & __GFP_CMA) && + gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE) alloc_flags |= ALLOC_CMA; trace_android_vh_alloc_flags_cma_adjust(gfp_mask, &alloc_flags); #endif