diff --git a/drivers/dma-buf/rk_heaps/Kconfig b/drivers/dma-buf/rk_heaps/Kconfig index 6ca3fbe765ee..e77267bd0322 100644 --- a/drivers/dma-buf/rk_heaps/Kconfig +++ b/drivers/dma-buf/rk_heaps/Kconfig @@ -35,6 +35,14 @@ config DMABUF_HEAPS_ROCKCHIP_CMA_ALIGNMENT If unsure, leave the default value "8". +config DMABUF_HEAPS_ROCKCHIP_SYSTEM_HEAP + tristate "DMA-BUF RockChip system Heap" + depends on DMABUF_HEAPS_ROCKCHIP + help + Choose this option to enable dma-buf RockChip SYSTEM heap. This heap is backed + by the system pages. If your system has these + regions, you should say Y here. + config DMABUF_RK_HEAPS_DEBUG bool "DMA-BUF RockChip Heap Debug" depends on DMABUF_HEAPS_ROCKCHIP diff --git a/drivers/dma-buf/rk_heaps/Makefile b/drivers/dma-buf/rk_heaps/Makefile index 30d44bb7d801..e0ba0351ea7b 100644 --- a/drivers/dma-buf/rk_heaps/Makefile +++ b/drivers/dma-buf/rk_heaps/Makefile @@ -4,3 +4,4 @@ rk-cma-heap-objs := rk-dma-cma.o rk-cma-heap.o obj-$(CONFIG_DMABUF_HEAPS_ROCKCHIP) += rk-dma-heap.o obj-$(CONFIG_DMABUF_HEAPS_ROCKCHIP_CMA_HEAP) += rk-cma-heap.o +obj-$(CONFIG_DMABUF_HEAPS_ROCKCHIP_SYSTEM_HEAP) += rk-system-heap.o diff --git a/drivers/dma-buf/rk_heaps/rk-dma-heap.c b/drivers/dma-buf/rk_heaps/rk-dma-heap.c index 1c68a7549e8a..26ba62f58d8f 100644 --- a/drivers/dma-buf/rk_heaps/rk-dma-heap.c +++ b/drivers/dma-buf/rk_heaps/rk-dma-heap.c @@ -215,6 +215,25 @@ void rk_dma_heap_total_dec(struct rk_dma_heap *heap, size_t len) mutex_unlock(&rk_heap_list_lock); } +int rk_dma_heap_alloc_pages(struct rk_dma_heap *heap, + struct page **pages, size_t len, gfp_t flags, + const char *name) +{ + len = PAGE_ALIGN(len); + if (!len) + return -EINVAL; + + return heap->ops->alloc_pages(heap, pages, len, flags, name); +} +EXPORT_SYMBOL_GPL(rk_dma_heap_alloc_pages); + +void rk_dma_heap_free_pages(struct rk_dma_heap *heap, + struct page **pages, unsigned int count) +{ + return heap->ops->free_pages(heap, pages, count); +} +EXPORT_SYMBOL_GPL(rk_dma_heap_free_pages); + static int rk_dma_heap_open(struct inode *inode, struct file *file) { struct rk_dma_heap *heap; @@ -398,7 +417,7 @@ struct rk_dma_heap *rk_dma_heap_add(const struct rk_dma_heap_export_info *exp_in return ERR_PTR(-EINVAL); } - if (!exp_info->ops || !exp_info->ops->allocate) { + if (!exp_info->ops || (!exp_info->permit_noalloc && !exp_info->ops->allocate)) { pr_err("rk_dma_heap: Cannot add heap with invalid ops struct\n"); return ERR_PTR(-EINVAL); } @@ -423,8 +442,10 @@ struct rk_dma_heap *rk_dma_heap_add(const struct rk_dma_heap_export_info *exp_in heap->support_cma = exp_info->support_cma; INIT_LIST_HEAD(&heap->dmabuf_list); INIT_LIST_HEAD(&heap->contig_list); + INIT_LIST_HEAD(&heap->pages_list); mutex_init(&heap->dmabuf_lock); mutex_init(&heap->contig_lock); + mutex_init(&heap->pages_lock); /* Find unused minor number */ ret = xa_alloc(&rk_dma_heap_minors, &minor, heap, @@ -450,7 +471,7 @@ struct rk_dma_heap *rk_dma_heap_add(const struct rk_dma_heap_export_info *exp_in NULL, heap->heap_devt, NULL, - heap->name); + "%s", heap->name); if (IS_ERR(heap->heap_dev)) { pr_err("rk_dma_heap: Unable to create device\n"); err_ret = ERR_CAST(heap->heap_dev); @@ -551,6 +572,21 @@ static int rk_dma_heap_dump_contig(void *data) return 0; } +static int rk_dma_heap_dump_pages(void *data) +{ + struct rk_dma_heap *heap = (struct rk_dma_heap *)data; + struct rk_dma_heap_pages_buf *buf; + + mutex_lock(&heap->pages_lock); + list_for_each_entry(buf, &heap->pages_list, node) { + seq_printf(heap->s, "\tAlloc by (%-20s)\t%lx (%lu KiB)\n", + buf->orig_alloc, buf->size, K(buf->size)); + } + mutex_unlock(&heap->pages_lock); + + return 0; +} + static ssize_t rk_total_pools_kb_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -616,6 +652,7 @@ static int rk_dma_heap_debug_show(struct seq_file *s, void *unused) heap->s = s; dma_buf_get_each(rk_dma_heap_dump_dmabuf, heap); rk_dma_heap_dump_contig(heap); + rk_dma_heap_dump_pages(heap); total += heap->total_size; } seq_printf(s, "\nTotal : 0x%lx (%lu KiB)\n", total, K(total)); @@ -665,6 +702,7 @@ static int rk_dma_heap_proc_show(struct seq_file *s, void *unused) heap->s = s; dma_buf_get_each(rk_dma_heap_dump_dmabuf, heap); rk_dma_heap_dump_contig(heap); + rk_dma_heap_dump_pages(heap); total += heap->total_size; } seq_printf(s, "\nTotal : 0x%lx (%lu KiB)\n", total, K(total)); diff --git a/drivers/dma-buf/rk_heaps/rk-dma-heap.h b/drivers/dma-buf/rk_heaps/rk-dma-heap.h index 11cce03e862d..88d2b1155aa2 100644 --- a/drivers/dma-buf/rk_heaps/rk-dma-heap.h +++ b/drivers/dma-buf/rk_heaps/rk-dma-heap.h @@ -35,7 +35,10 @@ struct rk_vmap_pfn_data { * struct rk_dma_heap_ops - ops to operate on a given heap * @allocate: allocate dmabuf and return struct dma_buf ptr * @get_pool_size: if heap maintains memory pools, get pool size in bytes - * + @alloc_contig_pages: alloc pages from CMA + @free_contig_pages: free pages to CMA + * @alloc_pages: alloc pages from system + * @free_pages: free pages to system * allocate returns dmabuf on success, ERR_PTR(-errno) on error. */ struct rk_dma_heap_ops { @@ -44,12 +47,16 @@ struct rk_dma_heap_ops { unsigned long fd_flags, unsigned long heap_flags, const char *name); + long (*get_pool_size)(struct rk_dma_heap *heap); struct page *(*alloc_contig_pages)(struct rk_dma_heap *heap, size_t len, const char *name); void (*free_contig_pages)(struct rk_dma_heap *heap, struct page *pages, size_t len, const char *name); - long (*get_pool_size)(struct rk_dma_heap *heap); + int (*alloc_pages)(struct rk_dma_heap *heap, struct page **pages, + size_t len, gfp_t flags, const char *name); + void (*free_pages)(struct rk_dma_heap *heap, + struct page **pages, unsigned int count); }; /** @@ -65,6 +72,7 @@ struct rk_dma_heap_export_info { const struct rk_dma_heap_ops *ops; void *priv; bool support_cma; + bool permit_noalloc; }; /** @@ -88,6 +96,8 @@ struct rk_dma_heap { struct mutex dmabuf_lock; struct list_head contig_list; /* contig buffer attach to this node */ struct mutex contig_lock; + struct list_head pages_list; + struct mutex pages_lock; struct cdev heap_cdev; struct kref refcount; struct device *heap_dev; @@ -112,6 +122,13 @@ struct rk_dma_heap_contig_buf { phys_addr_t end; }; +struct rk_dma_heap_pages_buf { + struct list_head node; + unsigned long size; + const char *orig_alloc; + phys_addr_t start; +}; + /** * rk_dma_heap_get_drvdata() - get per-heap driver data * @heap: DMA-Heap to retrieve private data for diff --git a/drivers/dma-buf/rk_heaps/rk-system-heap.c b/drivers/dma-buf/rk_heaps/rk-system-heap.c new file mode 100644 index 000000000000..1ee656445ae2 --- /dev/null +++ b/drivers/dma-buf/rk_heaps/rk-system-heap.c @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DMABUF System heap exporter + * + * Copyright (C) 2011 Google, Inc. + * Copyright (C) 2012, 2019, 2020 Linaro Ltd. + * + * Also utilizing parts of Andrew Davis' SRAM heap: + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ + * Andrew F. Davis + * + * Copyright (C) 2025 Rockchip Electronics Co., Ltd. + * Author: Simon Xue + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../mm/cma.h" +#include "rk-dma-heap.h" + +struct rk_system_heap { + struct rk_dma_heap *heap; +}; + +static int rk_system_heap_remove_pages_list(struct rk_dma_heap *heap, + struct page *page) +{ + struct rk_dma_heap_pages_buf *buf; + + mutex_lock(&heap->pages_lock); + list_for_each_entry(buf, &heap->pages_list, node) { + if (buf->start == page_to_phys(page)) { + dma_heap_print("<%s> free pages %ld to system\n", + buf->orig_alloc, buf->size); + list_del(&buf->node); + kfree(buf->orig_alloc); + kfree(buf); + break; + } + } + mutex_unlock(&heap->pages_lock); + + return 0; +} + +static int rk_system_heap_add_pages_list(struct rk_dma_heap *heap, + struct page *first_page, + unsigned long size, const char *name) +{ + struct rk_dma_heap_pages_buf *buf; + const char *name_tmp; + + buf = kzalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return -ENOMEM; + + INIT_LIST_HEAD(&buf->node); + if (!name) + name_tmp = current->comm; + else + name_tmp = name; + + buf->orig_alloc = kstrndup(name_tmp, RK_DMA_HEAP_NAME_LEN, GFP_KERNEL); + if (!buf->orig_alloc) { + kfree(buf); + return -ENOMEM; + } + + buf->size = size; + buf->start = page_to_phys(first_page); + + mutex_lock(&heap->pages_lock); + list_add_tail(&buf->node, &heap->pages_list); + mutex_unlock(&heap->pages_lock); + + dma_heap_print("<%s> alloc %ld from system\n", buf->orig_alloc, size); + + return 0; +} + + +static int rk_system_heap_allocate_pages(struct rk_dma_heap *heap, + struct page **pages, + size_t size, gfp_t flags, + const char *name) +{ + int ret; + unsigned int last_page = 0; + struct page *first_page = NULL; + unsigned long num_pages = size >> PAGE_SHIFT; + size_t raw_size = size; + + while (size > 0) { + struct page *page; + int order; + int i; + + order = get_order(size); + /* Don't over allocate*/ + if ((PAGE_SIZE << order) > size && order > 0) + order--; + + page = NULL; + while (!page) { + page = alloc_pages(GFP_KERNEL | __GFP_ZERO | + __GFP_NOWARN | flags, order); + if (page) { + if (!first_page) + first_page = page; + break; + } + + if (order == 0) { + while (last_page--) + __free_page(pages[last_page]); + return -ENOMEM; + } + order--; + } + + split_page(page, order); + for (i = 0; i < (1 << order); i++) + pages[last_page++] = &page[i]; + + size -= PAGE_SIZE << order; + } + + ret = rk_system_heap_add_pages_list(heap, first_page, raw_size, name); + if (ret) + goto free_pages; + + rk_dma_heap_total_inc(heap, raw_size); + + return 0; + +free_pages: + while (num_pages--) { + __free_page(pages[num_pages]); + pages[num_pages] = NULL; + } + + return ret; +} + +static void rk_system_heap_free_pages(struct rk_dma_heap *heap, + struct page **pages, unsigned int num_pages) +{ + /* Need more reasonable way */ + rk_system_heap_remove_pages_list(heap, pages[0]); + + rk_dma_heap_total_dec(heap, num_pages << PAGE_SHIFT); + + while (num_pages--) { + __free_page(pages[num_pages]); + pages[num_pages] = NULL; + } +} + +static const struct rk_dma_heap_ops rk_system_heap_ops = { + .alloc_pages = rk_system_heap_allocate_pages, + .free_pages = rk_system_heap_free_pages, +}; + +static int set_heap_dev_dma(struct device *heap_dev) +{ + int err = 0; + + if (!heap_dev) + return -EINVAL; + + dma_coerce_mask_and_coherent(heap_dev, DMA_BIT_MASK(64)); + + if (!heap_dev->dma_parms) { + heap_dev->dma_parms = devm_kzalloc(heap_dev, + sizeof(*heap_dev->dma_parms), + GFP_KERNEL); + if (!heap_dev->dma_parms) + return -ENOMEM; + + err = dma_set_max_seg_size(heap_dev, (unsigned int)DMA_BIT_MASK(64)); + if (err) { + devm_kfree(heap_dev, heap_dev->dma_parms); + dev_err(heap_dev, "Failed to set DMA segment size, err:%d\n", err); + return err; + } + } + + return 0; +} + +static int __rk_add_system_heap(void) +{ + struct rk_system_heap *sytem_heap; + struct rk_dma_heap_export_info exp_info; + int ret; + + sytem_heap = kzalloc(sizeof(*sytem_heap), GFP_KERNEL); + if (!sytem_heap) + return -ENOMEM; + + exp_info.name = "rk-system-heap"; + exp_info.ops = &rk_system_heap_ops; + exp_info.priv = sytem_heap; + exp_info.permit_noalloc = true; + + sytem_heap->heap = rk_dma_heap_add(&exp_info); + if (IS_ERR(sytem_heap->heap)) { + ret = PTR_ERR(sytem_heap->heap); + + kfree(sytem_heap); + return ret; + } + + ret = set_heap_dev_dma(rk_dma_heap_get_dev(sytem_heap->heap)); + if (ret) { + rk_dma_heap_put(sytem_heap->heap); + kfree(sytem_heap); + return ret; + } + + return 0; +} + +static int __init rk_add_system_heap(void) +{ + return __rk_add_system_heap(); +} + +#if defined(CONFIG_VIDEO_ROCKCHIP_THUNDER_BOOT_ISP) && !defined(CONFIG_INITCALL_ASYNC) +subsys_initcall(rk_add_system_heap); +#else +module_init(rk_add_system_heap); +#endif + +MODULE_DESCRIPTION("RockChip DMA-BUF SYSTEM Heap"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/rk-dma-heap.h b/include/linux/rk-dma-heap.h index f9b06cc18068..75292ac1ad12 100644 --- a/include/linux/rk-dma-heap.h +++ b/include/linux/rk-dma-heap.h @@ -100,6 +100,26 @@ struct page *rk_dma_heap_alloc_contig_pages(struct rk_dma_heap *heap, void rk_dma_heap_free_contig_pages(struct rk_dma_heap *heap, struct page *pages, size_t len, const char *name); +/** + * rk_dma_heap_alloc_pages - Allocate pages from system + * @heap: dma_heap for debug + * @pages: pages where to store + * @len: size to allocate + * @flags: flags used to alloc page + * @name: the name who allocate + */ +int rk_dma_heap_alloc_pages(struct rk_dma_heap *heap, + struct page **pages, size_t len, gfp_t flags, + const char *name); + +/** + * rk_dma_heap_free_pages - Free pages to system + * @heap: dma_heap for debug + * @pages: pages to free to system + * @count: page count to free + */ +void rk_dma_heap_free_pages(struct rk_dma_heap *heap, + struct page **pages, unsigned int count); #else static inline int rk_dma_heap_set_dev(struct device *heap_dev) { @@ -145,5 +165,17 @@ static inline void rk_dma_heap_free_contig_pages(struct rk_dma_heap *heap, struc size_t len, const char *name) { } + +static inline int rk_dma_heap_alloc_pages(struct rk_dma_heap *heap, + struct page **pages, size_t len, + gfp_t flags, const char *name) +{ + return -ENODEV; +} + +static inline void rk_dma_heap_free_pages(struct rk_dma_heap *heap, + struct page **pages, unsigned int count) +{ +} #endif #endif /* _DMA_HEAPS_H */