kernel_common_drivers/drivers/memory_ext/memory.c

// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
/*
 * Copyright (c) 2019 Amlogic, Inc. All rights reserved.
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/jiffies.h>
#include <linux/memblock.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/kasan.h>
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/amlogic/aml_cma.h>
#include <../../../mm/internal.h>
#include <linux/kmemleak.h>
#if IS_BUILTIN(CONFIG_AMLOGIC_PAGE_TRACE)
#include <linux/amlogic/page_trace.h>
#include <linux/kasan.h>
#endif

void should_wakeup_kswap(gfp_t gfp_mask, int order,
				       struct alloc_context *ac)
{
	unsigned long free_pages;
	struct zoneref *z = ac->preferred_zoneref;
	struct zone *zone;
	unsigned long high_wm;

	/*
	 * 1, if flag not allow reclaim
	 * 2, if with aotimic, we still need enable pre-wake up of
	 *    kswap to avoid large amount memory request fail in very
	 *    short time
	 */
	if (!(gfp_mask & __GFP_RECLAIM) && !(gfp_mask & GFP_ATOMIC))
		return;

	for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
					ac->nodemask) {
		free_pages = zone_page_state(zone, NR_FREE_PAGES);
	#ifdef CONFIG_AMLOGIC_CMA
		if (!can_use_cma(gfp_mask))
			free_pages -= zone_page_state(zone, NR_FREE_CMA_PAGES);
	#endif /* CONFIG_AMLOGIC_CMA */
		/*
		 * wake up kswapd before get pages from buddy, this help to
		 * fast reclaim process and can avoid memory become too low
		 * some times
		 */
		high_wm = high_wmark_pages(zone);
		if (gfp_mask & __GFP_HIGH) /* 1.5x if __GFP_HIGH */
			high_wm = ((high_wm * 3) / 2);
		if (free_pages <= high_wm)
			wakeup_kswapd(zone, gfp_mask, order, ac->highest_zoneidx);
	}
}
EXPORT_SYMBOL(should_wakeup_kswap);

#ifdef CONFIG_AMLOGIC_MEMORY_SLAB_LARGE
void adjust_redzone_end(const void *ptr, size_t size, unsigned long *p_end)
{
	if (PageOwnerPriv1(virt_to_page(ptr)))
		*p_end = (unsigned long)ptr + PAGE_ALIGN(size);
}

void *aml_slub_alloc_large(int node, size_t size, gfp_t flags, int order)
{
	struct page *page, *p;

	flags &= ~__GFP_COMP;
	page = alloc_pages_node(node, flags, order);
	if (page) {
		unsigned long used_pages = PAGE_ALIGN(size) / PAGE_SIZE;
		unsigned long total_pages = 1 << order;
		unsigned long saved = 0;
		unsigned long fun = 0;
		int i;

		/* record how many pages in first page*/
		__SetPageHead(page);
		SetPageOwnerPriv1(page); /* special flag */

	#if IS_BUILTIN(CONFIG_AMLOGIC_PAGE_TRACE)
		fun = get_page_trace(page);
	#endif

	#ifdef CONFIG_MEMCG
		if (memcg_kmem_online() && PageMemcgKmem(page)) {
			for (i = 1; i < total_pages; i++) {
				p = page + i;
				p->memcg_data = page->memcg_data;
				__memcg_kmem_charge_page(p, flags, 0);
			}
		}
	#endif

		for (i = 1; i < used_pages; i++) {
			p = page + i;
			set_compound_head(p, page);
		#if IS_BUILTIN(CONFIG_AMLOGIC_PAGE_TRACE)
			set_page_trace(page, 0, flags, (void *)fun);
		#endif
		}
		page->index = used_pages;
		split_page(page, order);
		p = page + used_pages;
		while (used_pages < total_pages) {
			__free_pages(p, 0);
			used_pages++;
			p++;
			saved++;
		}
		pr_debug("%s, page:%p, all:%5ld, size:%5ld, save:%5ld, f:%ps\n",
			__func__, page_address(page), total_pages * PAGE_SIZE,
			(long)size, saved * PAGE_SIZE, (void *)fun);
		return page;
	}
	return NULL;
}

static void aml_slub_free_large(struct folio *folio, const void *obj)
{
	unsigned int nr_pages, i;
	struct page *page = folio_page(folio, 0);

	if (page) {
		__ClearPageHead(page);
		ClearPageOwnerPriv1(page);
		nr_pages = page->index;
		pr_debug("%s, page:%p, pages:%d, obj:%p\n",
			__func__, page_address(page), nr_pages, obj);

		if (!nr_pages) {
			pr_err("%s index num error\n", __func__);
			dump_stack();
		}

		lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
			-(nr_pages * PAGE_SIZE));

		for (i = 0; i < nr_pages; i++)  {
			__free_pages(page, 0);
			page++;
		}
	}
}

int aml_free_nonslab_page(struct folio *folio, void *object)
{
	unsigned int order = folio_order(folio);

	/*
	 *if (WARN_ON_ONCE(order == 0))
	 *	pr_warn_once("object pointer: 0x%p\n", object);
	 */

	kmemleak_free(object);
	kasan_kfree_large(object);
	kmsan_kfree_large(object);

	if (unlikely(PageOwnerPriv1(folio_page(folio, 0)))) {
		aml_slub_free_large(folio, object);

		return 1;
	}

	lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
			      -(PAGE_SIZE << order));

	return 0;
}
#endif