dma-buf: rk_heaps: add rk-system-heap

Change-Id: Iebcb584b76f528140a8621a33bf554f413075abd
Signed-off-by: Simon Xue <xxm@rock-chips.com>

drivers/dma-buf/rk_heaps/Kconfig

@@ -35,6 +35,14 @@ config DMABUF_HEAPS_ROCKCHIP_CMA_ALIGNMENT
 
 	  If unsure, leave the default value "8".
 
+config DMABUF_HEAPS_ROCKCHIP_SYSTEM_HEAP
+	tristate "DMA-BUF RockChip system Heap"
+	depends on DMABUF_HEAPS_ROCKCHIP
+	help
+	  Choose this option to enable dma-buf RockChip SYSTEM heap. This heap is backed
+	  by the system pages. If your system has these
+	  regions, you should say Y here.
+
 config DMABUF_RK_HEAPS_DEBUG
 	bool "DMA-BUF RockChip Heap Debug"
 	depends on DMABUF_HEAPS_ROCKCHIP

drivers/dma-buf/rk_heaps/Makefile

@@ -4,3 +4,4 @@ rk-cma-heap-objs := rk-dma-cma.o rk-cma-heap.o
 
 obj-$(CONFIG_DMABUF_HEAPS_ROCKCHIP) += rk-dma-heap.o
 obj-$(CONFIG_DMABUF_HEAPS_ROCKCHIP_CMA_HEAP) += rk-cma-heap.o
+obj-$(CONFIG_DMABUF_HEAPS_ROCKCHIP_SYSTEM_HEAP) += rk-system-heap.o

drivers/dma-buf/rk_heaps/rk-dma-heap.c

@@ -215,6 +215,25 @@ void rk_dma_heap_total_dec(struct rk_dma_heap *heap, size_t len)
 	mutex_unlock(&rk_heap_list_lock);
 }
 
+int rk_dma_heap_alloc_pages(struct rk_dma_heap *heap,
+			    struct page **pages, size_t len, gfp_t flags,
+			    const char *name)
+{
+	len = PAGE_ALIGN(len);
+	if (!len)
+		return -EINVAL;
+
+	return heap->ops->alloc_pages(heap, pages, len, flags, name);
+}
+EXPORT_SYMBOL_GPL(rk_dma_heap_alloc_pages);
+
+void rk_dma_heap_free_pages(struct rk_dma_heap *heap,
+			    struct page **pages, unsigned int count)
+{
+	return heap->ops->free_pages(heap, pages, count);
+}
+EXPORT_SYMBOL_GPL(rk_dma_heap_free_pages);
+
 static int rk_dma_heap_open(struct inode *inode, struct file *file)
 {
 	struct rk_dma_heap *heap;
@@ -398,7 +417,7 @@ struct rk_dma_heap *rk_dma_heap_add(const struct rk_dma_heap_export_info *exp_in
 		return ERR_PTR(-EINVAL);
 	}
 
-	if (!exp_info->ops || !exp_info->ops->allocate) {
+	if (!exp_info->ops || (!exp_info->permit_noalloc && !exp_info->ops->allocate)) {
 		pr_err("rk_dma_heap: Cannot add heap with invalid ops struct\n");
 		return ERR_PTR(-EINVAL);
 	}
@@ -423,8 +442,10 @@ struct rk_dma_heap *rk_dma_heap_add(const struct rk_dma_heap_export_info *exp_in
 	heap->support_cma = exp_info->support_cma;
 	INIT_LIST_HEAD(&heap->dmabuf_list);
 	INIT_LIST_HEAD(&heap->contig_list);
+	INIT_LIST_HEAD(&heap->pages_list);
 	mutex_init(&heap->dmabuf_lock);
 	mutex_init(&heap->contig_lock);
+	mutex_init(&heap->pages_lock);
 
 	/* Find unused minor number */
 	ret = xa_alloc(&rk_dma_heap_minors, &minor, heap,
@@ -450,7 +471,7 @@ struct rk_dma_heap *rk_dma_heap_add(const struct rk_dma_heap_export_info *exp_in
 				       NULL,
 				       heap->heap_devt,
 				       NULL,
-				       heap->name);
+				       "%s", heap->name);
 	if (IS_ERR(heap->heap_dev)) {
 		pr_err("rk_dma_heap: Unable to create device\n");
 		err_ret = ERR_CAST(heap->heap_dev);
@@ -551,6 +572,21 @@ static int rk_dma_heap_dump_contig(void *data)
 	return 0;
 }
 
+static int rk_dma_heap_dump_pages(void *data)
+{
+	struct rk_dma_heap *heap = (struct rk_dma_heap *)data;
+	struct rk_dma_heap_pages_buf *buf;
+
+	mutex_lock(&heap->pages_lock);
+	list_for_each_entry(buf, &heap->pages_list, node) {
+		seq_printf(heap->s, "\tAlloc by (%-20s)\t%lx (%lu KiB)\n",
+			   buf->orig_alloc, buf->size, K(buf->size));
+	}
+	mutex_unlock(&heap->pages_lock);
+
+	return 0;
+}
+
 static ssize_t rk_total_pools_kb_show(struct kobject *kobj,
 				      struct kobj_attribute *attr, char *buf)
 {
@@ -616,6 +652,7 @@ static int rk_dma_heap_debug_show(struct seq_file *s, void *unused)
 		heap->s = s;
 		dma_buf_get_each(rk_dma_heap_dump_dmabuf, heap);
 		rk_dma_heap_dump_contig(heap);
+		rk_dma_heap_dump_pages(heap);
 		total += heap->total_size;
 	}
 	seq_printf(s, "\nTotal : 0x%lx (%lu KiB)\n", total, K(total));
@@ -665,6 +702,7 @@ static int rk_dma_heap_proc_show(struct seq_file *s, void *unused)
 		heap->s = s;
 		dma_buf_get_each(rk_dma_heap_dump_dmabuf, heap);
 		rk_dma_heap_dump_contig(heap);
+		rk_dma_heap_dump_pages(heap);
 		total += heap->total_size;
 	}
 	seq_printf(s, "\nTotal : 0x%lx (%lu KiB)\n", total, K(total));

drivers/dma-buf/rk_heaps/rk-dma-heap.h

@@ -35,7 +35,10 @@ struct rk_vmap_pfn_data {
  * struct rk_dma_heap_ops - ops to operate on a given heap
  * @allocate:		allocate dmabuf and return struct dma_buf ptr
  * @get_pool_size:	if heap maintains memory pools, get pool size in bytes
- *
+   @alloc_contig_pages: alloc pages from CMA
+   @free_contig_pages:	free pages to CMA
+ * @alloc_pages:	alloc pages from system
+ * @free_pages:		free pages to system
  * allocate returns dmabuf on success, ERR_PTR(-errno) on error.
  */
 struct rk_dma_heap_ops {
@@ -44,12 +47,16 @@ struct rk_dma_heap_ops {
 			unsigned long fd_flags,
 			unsigned long heap_flags,
 			const char *name);
+	long (*get_pool_size)(struct rk_dma_heap *heap);
 	struct page *(*alloc_contig_pages)(struct rk_dma_heap *heap,
 					   size_t len, const char *name);
 	void (*free_contig_pages)(struct rk_dma_heap *heap,
 				  struct page *pages, size_t len,
 				  const char *name);
-	long (*get_pool_size)(struct rk_dma_heap *heap);
+	int (*alloc_pages)(struct rk_dma_heap *heap, struct page **pages,
+			   size_t len, gfp_t flags, const char *name);
+	void (*free_pages)(struct rk_dma_heap *heap,
+			   struct page **pages, unsigned int count);
 };
 
 /**
@@ -65,6 +72,7 @@ struct rk_dma_heap_export_info {
 	const struct rk_dma_heap_ops *ops;
 	void *priv;
 	bool support_cma;
+	bool permit_noalloc;
 };
 
 /**
@@ -88,6 +96,8 @@ struct rk_dma_heap {
 	struct mutex dmabuf_lock;
 	struct list_head contig_list; /* contig buffer attach to this node */
 	struct mutex contig_lock;
+	struct list_head pages_list;
+	struct mutex pages_lock;
 	struct cdev heap_cdev;
 	struct kref refcount;
 	struct device *heap_dev;
@@ -112,6 +122,13 @@ struct rk_dma_heap_contig_buf {
 	phys_addr_t end;
 };
 
+struct rk_dma_heap_pages_buf {
+	struct list_head node;
+	unsigned long size;
+	const char *orig_alloc;
+	phys_addr_t start;
+};
+
 /**
  * rk_dma_heap_get_drvdata() - get per-heap driver data
  * @heap: DMA-Heap to retrieve private data for

drivers/dma-buf/rk_heaps/rk-system-heap.c

@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DMABUF System heap exporter
+ *
+ * Copyright (C) 2011 Google, Inc.
+ * Copyright (C) 2012, 2019, 2020 Linaro Ltd.
+ *
+ * Also utilizing parts of Andrew Davis' SRAM heap:
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
+ *	Andrew F. Davis <afd@ti.com>
+ *
+ * Copyright (C) 2025 Rockchip Electronics Co., Ltd.
+ * Author: Simon Xue <xxm@rock-chips.com>
+ */
+
+#include <linux/cma.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-map-ops.h>
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <uapi/linux/rk-dma-heap.h>
+#include <linux/proc_fs.h>
+#include "../../../mm/cma.h"
+#include "rk-dma-heap.h"
+
+struct rk_system_heap {
+	struct rk_dma_heap *heap;
+};
+
+static int rk_system_heap_remove_pages_list(struct rk_dma_heap *heap,
+					    struct page *page)
+{
+	struct rk_dma_heap_pages_buf *buf;
+
+	mutex_lock(&heap->pages_lock);
+	list_for_each_entry(buf, &heap->pages_list, node) {
+		if (buf->start == page_to_phys(page)) {
+			dma_heap_print("<%s> free pages %ld to system\n",
+				       buf->orig_alloc, buf->size);
+			list_del(&buf->node);
+			kfree(buf->orig_alloc);
+			kfree(buf);
+			break;
+		}
+	}
+	mutex_unlock(&heap->pages_lock);
+
+	return 0;
+}
+
+static int rk_system_heap_add_pages_list(struct rk_dma_heap *heap,
+					 struct page *first_page,
+					 unsigned long size, const char *name)
+{
+	struct rk_dma_heap_pages_buf *buf;
+	const char *name_tmp;
+
+	buf = kzalloc(sizeof(*buf), GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&buf->node);
+	if (!name)
+		name_tmp = current->comm;
+	else
+		name_tmp = name;
+
+	buf->orig_alloc = kstrndup(name_tmp, RK_DMA_HEAP_NAME_LEN, GFP_KERNEL);
+	if (!buf->orig_alloc) {
+		kfree(buf);
+		return -ENOMEM;
+	}
+
+	buf->size = size;
+	buf->start = page_to_phys(first_page);
+
+	mutex_lock(&heap->pages_lock);
+	list_add_tail(&buf->node, &heap->pages_list);
+	mutex_unlock(&heap->pages_lock);
+
+	dma_heap_print("<%s> alloc %ld from system\n", buf->orig_alloc, size);
+
+	return 0;
+}
+
+
+static int rk_system_heap_allocate_pages(struct rk_dma_heap *heap,
+					 struct page **pages,
+					 size_t size, gfp_t flags,
+					 const char *name)
+{
+	int ret;
+	unsigned int last_page = 0;
+	struct page *first_page = NULL;
+	unsigned long num_pages = size >> PAGE_SHIFT;
+	size_t raw_size = size;
+
+	while (size > 0) {
+		struct page *page;
+		int order;
+		int i;
+
+		order = get_order(size);
+		/* Don't over allocate*/
+		if ((PAGE_SIZE << order) > size && order > 0)
+			order--;
+
+		page = NULL;
+		while (!page) {
+			page = alloc_pages(GFP_KERNEL | __GFP_ZERO |
+					__GFP_NOWARN | flags, order);
+			if (page) {
+				if (!first_page)
+					first_page = page;
+				break;
+			}
+
+			if (order == 0) {
+				while (last_page--)
+					__free_page(pages[last_page]);
+				return -ENOMEM;
+			}
+			order--;
+		}
+
+		split_page(page, order);
+		for (i = 0; i < (1 << order); i++)
+			pages[last_page++] = &page[i];
+
+		size -= PAGE_SIZE << order;
+	}
+
+	ret = rk_system_heap_add_pages_list(heap, first_page, raw_size, name);
+	if (ret)
+		goto free_pages;
+
+	rk_dma_heap_total_inc(heap, raw_size);
+
+	return 0;
+
+free_pages:
+	while (num_pages--) {
+		__free_page(pages[num_pages]);
+		pages[num_pages] = NULL;
+	}
+
+	return ret;
+}
+
+static void rk_system_heap_free_pages(struct rk_dma_heap *heap,
+				   struct page **pages, unsigned int num_pages)
+{
+	/* Need more reasonable way */
+	rk_system_heap_remove_pages_list(heap, pages[0]);
+
+	rk_dma_heap_total_dec(heap, num_pages << PAGE_SHIFT);
+
+	while (num_pages--) {
+		__free_page(pages[num_pages]);
+		pages[num_pages] = NULL;
+	}
+}
+
+static const struct rk_dma_heap_ops rk_system_heap_ops = {
+	.alloc_pages = rk_system_heap_allocate_pages,
+	.free_pages = rk_system_heap_free_pages,
+};
+
+static int set_heap_dev_dma(struct device *heap_dev)
+{
+	int err = 0;
+
+	if (!heap_dev)
+		return -EINVAL;
+
+	dma_coerce_mask_and_coherent(heap_dev, DMA_BIT_MASK(64));
+
+	if (!heap_dev->dma_parms) {
+		heap_dev->dma_parms = devm_kzalloc(heap_dev,
+						   sizeof(*heap_dev->dma_parms),
+						   GFP_KERNEL);
+		if (!heap_dev->dma_parms)
+			return -ENOMEM;
+
+		err = dma_set_max_seg_size(heap_dev, (unsigned int)DMA_BIT_MASK(64));
+		if (err) {
+			devm_kfree(heap_dev, heap_dev->dma_parms);
+			dev_err(heap_dev, "Failed to set DMA segment size, err:%d\n", err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static int __rk_add_system_heap(void)
+{
+	struct rk_system_heap *sytem_heap;
+	struct rk_dma_heap_export_info exp_info;
+	int ret;
+
+	sytem_heap = kzalloc(sizeof(*sytem_heap), GFP_KERNEL);
+	if (!sytem_heap)
+		return -ENOMEM;
+
+	exp_info.name = "rk-system-heap";
+	exp_info.ops = &rk_system_heap_ops;
+	exp_info.priv = sytem_heap;
+	exp_info.permit_noalloc = true;
+
+	sytem_heap->heap = rk_dma_heap_add(&exp_info);
+	if (IS_ERR(sytem_heap->heap)) {
+		ret = PTR_ERR(sytem_heap->heap);
+
+		kfree(sytem_heap);
+		return ret;
+	}
+
+	ret = set_heap_dev_dma(rk_dma_heap_get_dev(sytem_heap->heap));
+	if (ret) {
+		rk_dma_heap_put(sytem_heap->heap);
+		kfree(sytem_heap);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int __init rk_add_system_heap(void)
+{
+	return __rk_add_system_heap();
+}
+
+#if defined(CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP) && !defined(CONFIG_INITCALL_ASYNC)
+subsys_initcall(rk_add_system_heap);
+#else
+module_init(rk_add_system_heap);
+#endif
+
+MODULE_DESCRIPTION("RockChip DMA-BUF SYSTEM Heap");
+MODULE_LICENSE("GPL");

include/linux/rk-dma-heap.h

@@ -100,6 +100,26 @@ struct page *rk_dma_heap_alloc_contig_pages(struct rk_dma_heap *heap,
 void rk_dma_heap_free_contig_pages(struct rk_dma_heap *heap, struct page *pages,
 				   size_t len, const char *name);
 
+/**
+ * rk_dma_heap_alloc_pages - Allocate pages from system
+ * @heap:	dma_heap for debug
+ * @pages:	pages where to store
+ * @len:	size to allocate
+ * @flags:	flags used to alloc page
+ * @name:	the name who allocate
+ */
+int rk_dma_heap_alloc_pages(struct rk_dma_heap *heap,
+			    struct page **pages, size_t len, gfp_t flags,
+			    const char *name);
+
+/**
+ * rk_dma_heap_free_pages - Free pages to system
+ * @heap:	dma_heap for debug
+ * @pages:	pages to free to system
+ * @count:	page count to free
+ */
+void rk_dma_heap_free_pages(struct rk_dma_heap *heap,
+			    struct page **pages, unsigned int count);
 #else
 static inline int rk_dma_heap_set_dev(struct device *heap_dev)
 {
@@ -145,5 +165,17 @@ static inline void rk_dma_heap_free_contig_pages(struct rk_dma_heap *heap, struc
 						 size_t len, const char *name)
 {
 }
+
+static inline int rk_dma_heap_alloc_pages(struct rk_dma_heap *heap,
+					struct page **pages, size_t len,
+					gfp_t flags, const char *name)
+{
+	return -ENODEV;
+}
+
+static inline void rk_dma_heap_free_pages(struct rk_dma_heap *heap,
+					struct page **pages, unsigned int count)
+{
+}
 #endif
 #endif /* _DMA_HEAPS_H */