mm: porting CMA optimization from 5.4 [1/2]

PD#SWPL-70022
PD#SWPL-131211

Problem:
No cma optimization on 5.15

Solution:
1, kernel only allow zram/anon pages use cma, new solution all
   all movable pages can use cma;
2, add __GFP_NO_CMAto tell buddy system which cases can't use
   cma;
3, keep high reference/active file cache pages when cma allocating;
4, hook real cma allocate/release interface to aml_cma.c, which can
   use boost mode when allocate large amount of cma;
5, count cma isolated pages to avoid deadloop congestion wait in kswapd
   or compaction case;
6, keep cma-unsuitable pages not compacte to cma area;
7, fix SWPL-131211

Verify:
local

Signed-off-by: Tao Zeng <tao.zeng@amlogic.com>
Change-Id: Ib82b898a596eba62e1aa1cb13719eab41f03ae58

arch/arm/mm/dma-mapping.c

@@ -709,6 +709,10 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
 	allowblock = gfpflags_allow_blocking(gfp);
 	cma = allowblock ? dev_get_cma_area(dev) : false;
 
+	#ifdef CONFIG_AMLOGIC_CMA
+	if (!!(gfp & __GFP_NO_CMA))
+		cma = false;
+	#endif
 	if (cma)
 		buf->allocator = &cma_allocator;
 	else if (is_coherent)

block/bdev.c

@@ -486,7 +486,11 @@ struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
 	inode->i_mode = S_IFBLK;
 	inode->i_rdev = 0;
 	inode->i_data.a_ops = &def_blk_aops;
+#ifdef CONFIG_AMLOGIC_CMA
+	mapping_set_gfp_mask(&inode->i_data, GFP_USER | __GFP_NO_CMA);
+#else
 	mapping_set_gfp_mask(&inode->i_data, GFP_USER);
+#endif
 
 	bdev = I_BDEV(inode);
 	mutex_init(&bdev->bd_fsfreeze_mutex);

drivers/usb/host/xhci.c

@@ -2072,7 +2072,11 @@ int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
 	 * process context, not interrupt context (or so documenation
 	 * for usb_set_interface() and usb_set_configuration() claim).
 	 */
+	#ifdef CONFIG_AMLOGIC_CMA
+	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO | __GFP_NO_CMA) < 0) {
+	#else
 	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
+	#endif
 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
 				__func__, ep->desc.bEndpointAddress);
 		return -ENOMEM;

include/linux/gfp.h

@@ -65,12 +65,19 @@ struct vm_area_struct;
 #endif
 #ifdef CONFIG_CMA
 #define ___GFP_CMA		0x8000000u
+#ifdef CONFIG_AMLOGIC_CMA
+#define ___GFP_NO_CMA		0x10000000u
+#endif /* CONFIG_AMLOGIC_CMA */
 #else
 #define ___GFP_CMA		0
 #endif
 #ifdef CONFIG_LOCKDEP
 #ifdef CONFIG_CMA
+#ifdef CONFIG_AMLOGIC_CMA
+#define ___GFP_NOLOCKDEP	0x20000000u
+#else
 #define ___GFP_NOLOCKDEP	0x10000000u
+#endif
 #else
 #define ___GFP_NOLOCKDEP	0x8000000u
 #endif
@@ -92,6 +99,9 @@ struct vm_area_struct;
 #define __GFP_DMA32	((__force gfp_t)___GFP_DMA32)
 #define __GFP_MOVABLE	((__force gfp_t)___GFP_MOVABLE)  /* ZONE_MOVABLE allowed */
 #define __GFP_CMA	((__force gfp_t)___GFP_CMA)
+#ifdef CONFIG_AMLOGIC_CMA
+#define __GFP_NO_CMA	((__force gfp_t)___GFP_NO_CMA)
+#endif
 #define GFP_ZONEMASK	(__GFP_DMA|__GFP_HIGHMEM|__GFP_DMA32|__GFP_MOVABLE)
 
 /**
@@ -280,7 +290,11 @@ struct vm_area_struct;
 #else
 #define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP))
 #endif
+#ifdef CONFIG_AMLOGIC_CMA
+#define __GFP_BITS_MASK ((__force gfp_t)((1 << (__GFP_BITS_SHIFT + 1)) - 1))
+#else
 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
+#endif
 
 /**
  * DOC: Useful GFP flag combinations

mm/cma.c

@@ -41,6 +41,16 @@
 
 #include "cma.h"
 
+#ifdef CONFIG_AMLOGIC_CMA
+#include <linux/amlogic/aml_cma.h>
+#include <linux/delay.h>
+#include <linux/sched/clock.h>
+#endif /* CONFIG_AMLOGIC_CMA */
+
+#ifdef CONFIG_AMLOGIC_SEC
+#include <linux/amlogic/secmon.h>
+#endif
+
 struct cma cma_areas[MAX_CMA_AREAS];
 unsigned cma_area_count;
 static DEFINE_MUTEX(cma_mutex);
@@ -155,9 +165,23 @@ static int __init cma_init_reserved_areas(void)
 	for (i = 0; i < cma_area_count; i++)
 		cma_activate_area(&cma_areas[i]);
 
+#ifdef CONFIG_AMLOGIC_SEC
+	/*
+	 * A73 cache speculate prefetch may cause SError when boot.
+	 * because it may prefetch cache line in secure memory range
+	 * which have already reserved by bootloader. So we must
+	 * clear mmu of secmon range before A73 core boot up
+	 */
+	secmon_clear_cma_mmu();
+#endif
 	return 0;
 }
+
+#ifdef CONFIG_AMLOGIC_CMA
+early_initcall(cma_init_reserved_areas);
+#else
 core_initcall(cma_init_reserved_areas);
+#endif
 
 /**
  * cma_init_reserved_mem() - create custom contiguous area from reserved memory
@@ -447,6 +471,14 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 	s64 ts;
 	struct cma_alloc_info cma_info = {0};
 	bool bypass = false;
+#ifdef CONFIG_AMLOGIC_CMA
+	int dummy;
+	unsigned long tick = 0;
+	unsigned long long in_tick, timeout;
+	int timeout_count = 0;
+
+	in_tick = sched_clock();
+#endif
 
 	trace_android_vh_cma_alloc_bypass(cma, count, align, gfp_mask,
 				&page, &bypass);
@@ -461,6 +493,13 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 	pr_debug("%s(cma %p, count %lu, align %d gfp_mask 0x%x)\n", __func__,
 			(void *)cma, count, align, gfp_mask);
 
+#ifdef CONFIG_AMLOGIC_CMA
+	cma_debug(0, NULL, "(cma %p, count %lu, align %d)\n",
+		  (void *)cma, count, align);
+	in_tick = sched_clock();
+	timeout = 2ULL * 1000000 * (1 + ((count * PAGE_SIZE) >> 20));
+#endif
+
 	if (!count)
 		goto out;
 
@@ -474,6 +513,10 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 	if (bitmap_count > bitmap_maxno)
 		goto out;
 
+#ifdef CONFIG_AMLOGIC_CMA
+	aml_cma_alloc_pre_hook(&dummy, count, &tick);
+#endif /* CONFIG_AMLOGIC_CMA */
+
 	for (;;) {
 		struct acr_info info = {0};
 
@@ -518,7 +561,11 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 
 		pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
 		mutex_lock(&cma_mutex);
+	#ifdef CONFIG_AMLOGIC_CMA
+		ret = aml_cma_alloc_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask);
+	#else
 		ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA, gfp_mask, &info);
+	#endif
 		cma_info.nr_migrated += info.nr_migrated;
 		cma_info.nr_reclaimed += info.nr_reclaimed;
 		cma_info.nr_mapped += info.nr_mapped;
@@ -547,6 +594,7 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 		trace_cma_alloc_busy_retry(cma->name, pfn, pfn_to_page(pfn),
 					   count, align);
 
+	#ifndef CONFIG_AMLOGIC_CMA
 		if (info.failed_pfn && gfp_mask & __GFP_NORETRY) {
 			/* try again from following failed page */
 			start = (pfn_max_align_up(info.failed_pfn + 1) -
@@ -556,8 +604,30 @@ struct page *cma_alloc(struct cma *cma, unsigned long count,
 			/* try again with a bit different memory target */
 			start = bitmap_no + mask + 1;
 		}
+	#else
+		/*
+		 * CMA allocation time out, for example:
+		 * 1. set isolation failed.
+		 * 2. refcout and mapcount mismatch.
+		 * may blocked on some pages, relax CPU and try later.
+		 */
+		if ((sched_clock() - in_tick) >= timeout) {
+			if (timeout_count > 20) {
+				pr_err("cma: %s alloc too long %lx, %lx\n",
+					cma->name, pfn, count);
+				cma_debug_level = 6;
+			}
+			timeout_count++;
+			usleep_range(1000, 2000);
+		}
+	#endif
 	}
 
+#ifdef CONFIG_AMLOGIC_CMA
+	if (cma_debug_level == 6)
+		cma_debug_level = 0;
+#endif
+
 	trace_cma_alloc_finish(cma->name, pfn, page, count, align);
 	trace_cma_alloc_info(cma->name, page, count, align, &cma_info);
 
@@ -588,6 +658,9 @@ out:
 		if (cma)
 			cma_sysfs_account_fail_pages(cma, count);
 	}
+#ifdef CONFIG_AMLOGIC_CMA
+	aml_cma_alloc_post_hook(&dummy, count, page, tick, ret);
+#endif
 
 	return page;
 }
@@ -620,7 +693,11 @@ bool cma_release(struct cma *cma, const struct page *pages,
 
 	VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);
 
+#ifdef CONFIG_AMLOGIC_CMA
+	aml_cma_free(pfn, count, 1);
+#else
 	free_contig_range(pfn, count);
+#endif
 	cma_clear_bitmap(cma, pfn, count);
 	trace_cma_release(cma->name, pfn, pages, count);
 

mm/compaction.c

@@ -27,6 +27,9 @@
 #ifdef CONFIG_AMLOGIC_PAGE_TRACE
 #include <linux/amlogic/page_trace.h>
 #endif
+#ifdef CONFIG_AMLOGIC_CMA
+#include <linux/amlogic/aml_cma.h>
+#endif
 
 #ifdef CONFIG_COMPACTION
 static inline void count_compact_event(enum vm_event_item item)
@@ -800,6 +803,9 @@ static bool too_many_isolated(pg_data_t *pgdat)
 	isolated = node_page_state(pgdat, NR_ISOLATED_FILE) +
 			node_page_state(pgdat, NR_ISOLATED_ANON);
 
+#ifdef CONFIG_AMLOGIC_CMA
+	check_cma_isolated(&isolated, active, inactive);
+#endif
 	return isolated > (inactive + active) / 2;
 }
 
@@ -907,6 +913,11 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 				cc->contended = true;
 				ret = -EINTR;
 
+			#ifdef CONFIG_AMLOGIC_CMA
+				if (cc->alloc_contig)
+					cma_debug(1, page, "abort by sig, low_pfn:%lx, swap:%ld\n",
+						  low_pfn, SWAP_CLUSTER_MAX);
+			#endif
 				goto fatal_pending;
 			}
 
@@ -927,6 +938,11 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 			if (!cc->ignore_skip_hint && get_pageblock_skip(page)) {
 				low_pfn = end_pfn;
 				page = NULL;
+			#ifdef CONFIG_AMLOGIC_CMA
+				if (cc->alloc_contig)
+					cma_debug(1, page, "abort by skip, low_pfn:%lx\n",
+						  low_pfn);
+			#endif
 				goto isolate_abort;
 			}
 			valid_page = page;
@@ -944,6 +960,11 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 				if (ret == -EBUSY)
 					ret = 0;
 				low_pfn += (1UL << compound_order(page)) - 1;
+			#ifdef CONFIG_AMLOGIC_CMA
+				if (cc->alloc_contig)
+					cma_debug(1, page, "abort by huge, low_pfn:%lx\n",
+						  low_pfn);
+			#endif
 				goto isolate_fail;
 			}
 
@@ -996,6 +1017,11 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 
 			if (likely(order < MAX_ORDER))
 				low_pfn += (1UL << order) - 1;
+		#ifdef CONFIG_AMLOGIC_CMA
+			if (cc->alloc_contig)
+				cma_debug(1, page, "abort by compound, low_pfn:%lx\n",
+					  low_pfn);
+		#endif
 			goto isolate_fail;
 		}
 
@@ -1020,6 +1046,11 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 					goto isolate_success;
 			}
 
+		#ifdef CONFIG_AMLOGIC_CMA
+			if (cc->alloc_contig && page_count(page))
+				cma_debug(1, page, "abort by LRU, low_pfn:%lx\n",
+					  low_pfn);
+		#endif
 			goto isolate_fail;
 		}
 
@@ -1028,8 +1059,17 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 		 * sure the page is not being freed elsewhere -- the
 		 * page release code relies on it.
 		 */
+	#ifdef CONFIG_AMLOGIC_CMA
+		if (unlikely(!get_page_unless_zero(page))) {
+			if (cc->alloc_contig)
+				cma_debug(1, page, "none zero ref, low_pfn:%lx\n",
+					  low_pfn);
+			goto isolate_fail;
+		}
+	#else
 		if (unlikely(!get_page_unless_zero(page)))
 			goto isolate_fail;
+	#endif
 
 		/*
 		 * Migration will fail if an anonymous page is pinned in memory,
@@ -1037,15 +1077,34 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 		 * admittedly racy check.
 		 */
 		mapping = page_mapping(page);
+	#ifdef CONFIG_AMLOGIC_CMA
+		if (!mapping && (page_count(page) - 1) > total_mapcount(page)) {
+			if (cc->alloc_contig)
+				cma_debug(1, page, "mc/rc miss match, low_pfn:%lx\n",
+					  low_pfn);
+			goto isolate_fail_put;
+		}
+		check_page_to_cma(cc, mapping, page);
+	#else
 		if (!mapping && (page_count(page) - 1) > total_mapcount(page))
 			goto isolate_fail_put;
+	#endif
 
 		/*
 		 * Only allow to migrate anonymous pages in GFP_NOFS context
 		 * because those do not depend on fs locks.
 		 */
+	#ifdef CONFIG_AMLOGIC_CMA
+		if (!(cc->gfp_mask & __GFP_FS) && mapping) {
+			if (cc->alloc_contig)
+				cma_debug(1, page, "no fs ctx, low_pfn:%lx\n",
+					  low_pfn);
+			goto isolate_fail_put;
+		}
+	#else
 		if (!(cc->gfp_mask & __GFP_FS) && mapping)
 			goto isolate_fail_put;
+	#endif
 
 		/* Only take pages on LRU: a check now makes later tests safe */
 		if (!PageLRU(page))
@@ -1087,8 +1146,17 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 		}
 
 		/* Try isolate the page */
+	#ifdef CONFIG_AMLOGIC_CMA
+		if (!TestClearPageLRU(page)) {
+			if (cc->alloc_contig)
+				cma_debug(1, page, "clear lru fail, low_pfn:%lx, mode:%x\n",
+					  low_pfn, mode);
+			goto isolate_fail_put;
+		}
+	#else
 		if (!TestClearPageLRU(page))
 			goto isolate_fail_put;
+	#endif
 
 		lruvec = mem_cgroup_page_lruvec(page);
 
@@ -1105,8 +1173,17 @@ isolate_migratepages_block(struct compact_control_ext *cc_ext, unsigned long low
 			/* Try get exclusive access under lock */
 			if (!skip_updated) {
 				skip_updated = true;
+			#ifdef CONFIG_AMLOGIC_CMA
+				if (test_and_set_skip(cc, page, low_pfn)) {
+					if (cc->alloc_contig)
+						cma_debug(1, page, "skip fail, low_pfn:%lx, mode:%x\n",
+							  low_pfn, mode);
+					goto isolate_abort;
+				}
+			#else
 				if (test_and_set_skip(cc, page, low_pfn))
 					goto isolate_abort;
+			#endif
 			}
 
 			/*
@@ -1613,6 +1690,9 @@ static void isolate_freepages(struct compact_control_ext *cc_ext)
 	struct list_head *freelist = &cc->freepages;
 	unsigned int stride;
 	bool bypass = false;
+#ifdef CONFIG_AMLOGIC_CMA
+	int migrate_type;
+#endif /* CONFIG_AMLOGIC_CMA */
 
 	/* Try a small search of the free lists for a candidate */
 	isolate_start_pfn = fast_isolate_freepages(cc);
@@ -1672,6 +1752,16 @@ static void isolate_freepages(struct compact_control_ext *cc_ext)
 		if (bypass)
 			continue;
 
+	#ifdef CONFIG_AMLOGIC_CMA
+		/* avoid compact to cma area */
+		migrate_type = get_pageblock_migratetype(page);
+		if (is_migrate_isolate(migrate_type))
+			continue;
+		if (is_migrate_cma(migrate_type) &&
+		    test_bit(FORBID_TO_CMA_BIT, &cc->total_migrate_scanned))
+			continue;
+	#endif /* CONFIG_AMLOGIC_CMA */
+
 		/* Found a block suitable for isolating free pages from. */
 		nr_isolated = isolate_freepages_block(cc, &isolate_start_pfn,
 					block_end_pfn, freelist, stride, false);
@@ -1738,9 +1828,13 @@ static struct page *compaction_alloc(struct page *migratepage,
 			return NULL;
 	}
 
+#ifdef CONFIG_AMLOGIC_CMA
+	freepage = get_compact_page(migratepage, cc);
+#else
 	freepage = list_entry(cc->freepages.next, struct page, lru);
 	list_del(&freepage->lru);
 	cc->nr_freepages--;
+#endif
 #ifdef CONFIG_AMLOGIC_PAGE_TRACE
 	replace_page_trace(freepage, migratepage);
 #endif
@@ -2559,6 +2653,9 @@ out:
 			cc->zone->compact_cached_free_pfn = free_pfn;
 	}
 
+#ifdef CONFIG_AMLOGIC_CMA
+	__clear_bit(FORBID_TO_CMA_BIT, &cc->total_migrate_scanned);
+#endif
 	count_compact_events(COMPACTMIGRATE_SCANNED, cc->total_migrate_scanned);
 	count_compact_events(COMPACTFREE_SCANNED, cc->total_free_scanned);
 

mm/internal.h

@@ -608,6 +608,9 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 #define ALLOC_NOFRAGMENT	  0x0
 #endif
 #define ALLOC_KSWAPD		0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
+#ifdef CONFIG_AMLOGIC_CMA
+#define ALLOC_MOVABLE_USE_CMA_FIRST	0x1000 /* preferred to allocate from cma for movable */
+#endif
 
 enum ttu_flags;
 struct tlbflush_unmap_batch;

mm/ksm.c

@@ -2125,6 +2125,17 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
 	}
 	tree_rmap_item =
 		unstable_tree_search_insert(rmap_item, page, &tree_page);
+#ifdef CONFIG_AMLOGIC_CMA
+	/*
+	 * Now page is inserted to unstable tree, but do not
+	 * let cma page to be kpage, it can be merged with other pages
+	 */
+	if (cma_page(page)) {
+		if (tree_rmap_item)
+			put_page(tree_page);
+		return;
+	}
+#endif /* CONFIG_AMLOGIC_CMA */
 	if (tree_rmap_item) {
 		bool split;
 

mm/memory.c

@@ -3183,13 +3183,25 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 	}
 
 	if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
+	#ifdef CONFIG_AMLOGIC_CMA
+		gfp_t tmp_flag = GFP_HIGHUSER_MOVABLE |
+				 __GFP_NO_CMA | __GFP_ZERO;
+
+		new_page = alloc_page_vma(tmp_flag, vma, vmaddr);
+	#else
 		new_page = alloc_zeroed_user_highpage_movable(vma,
 							      vmf->address);
+	#endif
 		if (!new_page)
 			goto out;
 	} else {
+	#ifdef CONFIG_AMLOGIC_CMA
+		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_NO_CMA,
+					  vma, vmf->address);
+	#else
 		new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
 				vmf->address);
+	#endif
 		if (!new_page)
 			goto out;
 
@@ -4939,6 +4951,12 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
 	p4d_t *p4d;
 	vm_fault_t ret;
 
+#ifdef CONFIG_AMLOGIC_CMA
+	if (vma->vm_file && vma->vm_file->f_mapping &&
+	    (vma->vm_flags & VM_EXEC))
+		vma->vm_file->f_mapping->gfp_mask |= __GFP_NO_CMA;
+#endif
+
 #ifdef CONFIG_SPECULATIVE_PAGE_FAULT
 	if (flags & FAULT_FLAG_SPECULATIVE) {
 		pgd_t pgdval;

mm/migrate.c

@@ -50,6 +50,10 @@
 #include <linux/ptrace.h>
 #include <linux/oom.h>
 #include <linux/memory.h>
+#ifdef CONFIG_AMLOGIC_CMA
+#include <linux/amlogic/aml_cma.h>
+#include <linux/delay.h>
+#endif
 
 #include <asm/tlbflush.h>
 
@@ -296,6 +300,9 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 	pte_t pte;
 	swp_entry_t entry;
 	struct page *page;
+#ifdef CONFIG_AMLOGIC_CMA
+	bool need_wait = 0;
+#endif
 
 	spin_lock(ptl);
 	pte = *ptep;
@@ -307,6 +314,17 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 		goto out;
 
 	page = pfn_swap_entry_to_page(entry);
+#ifdef CONFIG_AMLOGIC_CMA
+	/* This page is under cma allocating, do not increase it ref */
+	if (in_cma_allocating(page)) {
+		pr_debug("%s, Page:%lx, flags:%lx, m:%d, c:%d, map:%p\n",
+			 __func__, page_to_pfn(page), page->flags,
+			page_mapcount(page), page_count(page),
+			page->mapping);
+		need_wait = 1;
+		goto out;
+	}
+#endif
 	page = compound_head(page);
 
 	/*
@@ -321,6 +339,10 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 	return;
 out:
 	pte_unmap_unlock(ptep, ptl);
+#ifdef CONFIG_AMLOGIC_CMA
+	if (need_wait)
+		usleep_range(1000, 1100);
+#endif
 }
 
 void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
@@ -392,6 +414,11 @@ int migrate_page_move_mapping(struct address_space *mapping,
 
 	if (!mapping) {
 		/* Anonymous page without mapping */
+	#ifdef CONFIG_AMLOGIC_CMA
+		if (page_count(page) != expected_count)
+			cma_debug(2, page, " anon page cnt miss match, e:%d\n",
+				  expected_count);
+	#endif
 		if (page_count(page) != expected_count)
 			return -EAGAIN;
 
@@ -410,11 +437,19 @@ int migrate_page_move_mapping(struct address_space *mapping,
 	xas_lock_irq(&xas);
 	if (page_count(page) != expected_count || xas_load(&xas) != page) {
 		xas_unlock_irq(&xas);
+	#ifdef CONFIG_AMLOGIC_CMA
+		cma_debug(2, page, " anon page cnt miss match, e:%d, p:%d\n",
+			  expected_count, page_has_private(page));
+	#endif
 		return -EAGAIN;
 	}
 
 	if (!page_ref_freeze(page, expected_count)) {
 		xas_unlock_irq(&xas);
+	#ifdef CONFIG_AMLOGIC_CMA
+		cma_debug(2, page, " page free fail, e:%d, p:%d\n",
+			  expected_count, page_has_private(page));
+	#endif
 		return -EAGAIN;
 	}
 
@@ -1533,6 +1568,9 @@ retry:
 
 				/* Hugetlb migration is unsupported */
 				nr_failed++;
+			#ifdef CONFIG_AMLOGIC_CMA
+				cma_debug(2, page, " NO SYS\n");
+			#endif
 				break;
 			case -ENOMEM:
 				/*
@@ -1551,6 +1589,9 @@ retry:
 					goto out;
 				}
 				nr_failed++;
+			#ifdef CONFIG_AMLOGIC_CMA
+				cma_debug(2, page, " NO MEM\n");
+			#endif
 				goto out;
 			case -EAGAIN:
 				if (is_thp) {
@@ -1580,6 +1621,9 @@ retry:
 					break;
 				}
 				nr_failed++;
+			#ifdef CONFIG_AMLOGIC_CMA
+				cma_debug(2, page, " failed:%d\n", rc);
+			#endif
 				break;
 			}
 		}

mm/page_alloc.c

@@ -85,6 +85,10 @@
 #include <linux/amlogic/page_trace.h>
 #endif
 
+#ifdef CONFIG_AMLOGIC_CMA
+#include <linux/amlogic/aml_cma.h>
+#endif
+
 EXPORT_TRACEPOINT_SYMBOL_GPL(mm_page_alloc);
 EXPORT_TRACEPOINT_SYMBOL_GPL(mm_page_free);
 
@@ -3168,6 +3172,18 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype,
 {
 	struct page *page;
 
+#ifdef CONFIG_AMLOGIC_CMA
+	/* use CMA first */
+	if (migratetype == MIGRATE_MOVABLE && alloc_flags & ALLOC_MOVABLE_USE_CMA_FIRST) {
+		page = __rmqueue_cma_fallback(zone, order);
+		if (page) {
+			trace_mm_page_alloc_zone_locked(page, order,
+							MIGRATE_CMA);
+			return page;
+		}
+	}
+#endif /* CONFIG_AMLOGIC_CMA */
+
 retry:
 	page = __rmqueue_smallest(zone, order, migratetype);
 
@@ -3180,6 +3196,31 @@ retry:
 	return page;
 }
 
+#ifdef CONFIG_AMLOGIC_CMA
+/*
+ * get page but not cma
+ */
+static struct page *rmqueue_no_cma(struct zone *zone, unsigned int order,
+				   int migratetype, unsigned int alloc_flags)
+{
+	struct page *page;
+
+	spin_lock(&zone->lock);
+retry:
+	page = __rmqueue_smallest(zone, order, migratetype);
+	if (unlikely(!page)) {
+		if (!page && __rmqueue_fallback(zone, order, migratetype, alloc_flags))
+			goto retry;
+	}
+	WARN_ON(page && is_migrate_cma(get_pcppage_migratetype(page)));
+	if (page)
+		__mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
+
+	spin_unlock(&zone->lock);
+	return page;
+}
+#endif /* CONFIG_AMLOGIC_CMA */
+
 #ifdef CONFIG_CMA
 static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
 				  int migratetype,
@@ -3890,6 +3931,12 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
 {
 	struct page *page = NULL;
 	struct list_head *list = NULL;
+#ifdef CONFIG_AMLOGIC_CMA
+	bool cma = can_use_cma(gfp_flags);
+
+	if (cma)
+		alloc_flags |= ALLOC_MOVABLE_USE_CMA_FIRST;
+#endif
 
 	do {
 		/* First try to get CMA pages */
@@ -3907,6 +3954,48 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
 		}
 
 		page = list_first_entry(list, struct page, pcp_list);
+	#ifdef CONFIG_AMLOGIC_CMA
+		/*
+		 * USING CMA FIRST POLICY situations:
+		 * 1. CMA pages may return to pcp and allocated next
+		 *    but gfp mask is not suitable for CMA;
+		 * 2. MOVABLE pages may return to pcp and allocated next
+		 *    but gfp mask is suitable for CMA
+		 *
+		 * For 1, we should replace a none-CMA page
+		 * For 2, we should replace with a cma page
+		 * before page is deleted from PCP list.
+		 */
+		if (!cma && is_migrate_cma_page(page)) {
+			/* case 1 */
+			page = rmqueue_no_cma(zone, 0, migratetype, alloc_flags);
+			if (page) {
+				check_new_pcp(page);
+				return page;
+			} else {
+				return NULL;
+			}
+		} else if ((migratetype == MIGRATE_MOVABLE) &&
+			   (get_pcppage_migratetype(page) != MIGRATE_CMA) &&
+			   cma) {
+			struct page *t;
+
+			spin_lock(&zone->lock);
+			t = __rmqueue_cma_fallback(zone, 0);
+			/* can't alloc cma pages or not ready */
+			if (!t || check_new_pcp(page)) {
+				spin_unlock(&zone->lock);
+				goto use_pcp;
+			}
+			page = t;
+			__mod_zone_freepage_state(zone, -(1),
+						  get_pcppage_migratetype(t));
+			spin_unlock(&zone->lock);
+			check_new_pcp(page);
+			return page;
+		}
+use_pcp:
+	#endif /* CONFIG_AMLOGIC_CMA */
 		list_del(&page->pcp_list);
 		pcp->count -= 1 << order;
 	} while (check_new_pcp(page));
@@ -3962,6 +4051,12 @@ struct page *rmqueue(struct zone *preferred_zone,
 			int migratetype)
 {
 	struct page *page;
+#ifdef CONFIG_AMLOGIC_CMA
+	bool cma = can_use_cma(gfp_flags);
+
+	if (cma)
+		alloc_flags |= ALLOC_MOVABLE_USE_CMA_FIRST;
+#endif
 
 	if (likely(pcp_allowed_order(order))) {
 		page = rmqueue_pcplist(preferred_zone, zone, order,
@@ -4121,6 +4216,9 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
 			min -= min / 4;
 	}
 
+#ifdef CONFIG_AMLOGIC_CMA
+	check_water_mark(free_pages, min + z->lowmem_reserve[highest_zoneidx]);
+#endif
 	/*
 	 * Check watermarks for an order-0 allocation request. If these
 	 * are not met, then a high-order request also cannot go ahead
@@ -5673,6 +5771,10 @@ struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
 	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
 	struct alloc_context ac = { };
 
+#ifdef CONFIG_AMLOGIC_CMA
+	update_gfp_flags(&gfp);
+#endif
+
 	trace_android_vh_alloc_pages_entry(&gfp, order, preferred_nid, nodemask);
 	/*
 	 * There are several places where we assume that the order value is sane

mm/readahead.c

@@ -202,6 +202,11 @@ void page_cache_ra_unbounded(struct readahead_control *ractl,
 	 */
 	unsigned int nofs = memalloc_nofs_save();
 
+#ifdef CONFIG_AMLOGIC_CMA
+	if (ractl && ractl->file &&
+	    (ractl->file->f_mode & (FMODE_WRITE | FMODE_WRITE_IOCTL)))
+		gfp_mask |= __GFP_WRITE;
+#endif /* CONFIG_AMLOGIC_CMA */
 	filemap_invalidate_lock_shared(mapping);
 	/*
 	 * Preallocate as many pages as we will need.

mm/shmem.c

@@ -1523,7 +1523,11 @@ static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
 	};
 
 	shmem_pseudo_vma_init(&pvma, info, index);
+#ifdef CONFIG_AMLOGIC_CMA
+	page = swap_cluster_readahead(swap, gfp | __GFP_NO_CMA, &vmf);
+#else
 	page = swap_cluster_readahead(swap, gfp, &vmf);
+#endif
 	shmem_pseudo_vma_destroy(&pvma);
 
 	return page;
@@ -1567,8 +1571,13 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp,
 		return NULL;
 
 	shmem_pseudo_vma_init(&pvma, info, hindex);
+#ifdef CONFIG_AMLOGIC_CMA
+	page = alloc_pages_vma(gfp | __GFP_NO_CMA, HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(),
+			       true);
+#else
 	page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(),
 			       true);
+#endif
 	shmem_pseudo_vma_destroy(&pvma);
 	if (page)
 		prep_transhuge_page(page);
@@ -1584,7 +1593,11 @@ static struct page *shmem_alloc_page(gfp_t gfp,
 	struct page *page;
 
 	shmem_pseudo_vma_init(&pvma, info, index);
+#ifdef CONFIG_AMLOGIC_CMA
+	page = alloc_page_vma(gfp | __GFP_NO_CMA, &pvma, 0);
+#else
 	page = alloc_page_vma(gfp, &pvma, 0);
+#endif
 	shmem_pseudo_vma_destroy(&pvma);
 
 	return page;

mm/swap_state.c

@@ -456,7 +456,11 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
 		 * before marking swap_map SWAP_HAS_CACHE, when -EEXIST will
 		 * cause any racers to loop around until we add it to cache.
 		 */
+	#ifdef CONFIG_AMLOGIC_CMA
+		page = alloc_page_vma(gfp_mask | __GFP_NO_CMA, vma, addr);
+	#else
 		page = alloc_page_vma(gfp_mask, vma, addr);
+	#endif
 		if (!page)
 			return NULL;
 

mm/vmscan.c

@@ -74,6 +74,10 @@
 #undef CREATE_TRACE_POINTS
 #include <trace/hooks/mm.h>
 
+#ifdef CONFIG_AMLOGIC_CMA
+#include <linux/amlogic/aml_cma.h>
+#endif
+
 EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin);
 EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end);
 
@@ -1875,13 +1879,21 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 	struct page *page, *next;
 	LIST_HEAD(clean_pages);
 	unsigned int noreclaim_flag;
+#ifdef CONFIG_AMLOGIC_CMA
+	LIST_HEAD(high_active_pages);
+#endif
 
 	list_for_each_entry_safe(page, next, page_list, lru) {
 		if (!PageHuge(page) && page_is_file_lru(page) &&
 		    !PageDirty(page) && !__PageMovable(page) &&
 		    !PageUnevictable(page)) {
+		#ifdef CONFIG_AMLOGIC_CMA
+			cma_keep_high_active(page, &high_active_pages,
+					     &clean_pages);
+		#else
 			ClearPageActive(page);
 			list_move(&page->lru, &clean_pages);
+		#endif
 		}
 	}
 
@@ -1897,6 +1909,9 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
 	memalloc_noreclaim_restore(noreclaim_flag);
 
 	list_splice(&clean_pages, page_list);
+#ifdef CONFIG_AMLOGIC_CMA
+	list_splice(&high_active_pages, page_list);
+#endif
 	mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
 			    -(long)nr_reclaimed);
 	/*
@@ -2144,6 +2159,9 @@ static int too_many_isolated(struct pglist_data *pgdat, int file,
 	if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
 		inactive >>= 3;
 
+#ifdef CONFIG_AMLOGIC_CMA
+	check_cma_isolated(&isolated, inactive, inactive);
+#endif
 	return isolated > inactive;
 }