From fdc1ada0ca79281e296ce81bf62c5fea959a0183 Mon Sep 17 00:00:00 2001 From: Yu Qiaowei Date: Wed, 22 Dec 2021 17:48:00 +0800 Subject: [PATCH] video: rockchip: rga3: Add rga_mm.c/rga_mm.h. 1. Add improt/release dma-buf interface. 2. Switch to using idr to manage import/release buffer. Signed-off-by: Yu Qiaowei Change-Id: I40930f3144037435a7c1b78affba2caf81075882 --- drivers/video/rockchip/rga3/Makefile | 2 +- drivers/video/rockchip/rga3/include/rga.h | 22 ++ .../video/rockchip/rga3/include/rga_dma_buf.h | 9 +- drivers/video/rockchip/rga3/include/rga_drv.h | 47 +++- drivers/video/rockchip/rga3/include/rga_mm.h | 39 +++ drivers/video/rockchip/rga3/rga_dma_buf.c | 60 ++++ drivers/video/rockchip/rga3/rga_drv.c | 133 +++++++++ drivers/video/rockchip/rga3/rga_mm.c | 256 ++++++++++++++++++ 8 files changed, 552 insertions(+), 16 deletions(-) create mode 100644 drivers/video/rockchip/rga3/include/rga_mm.h create mode 100644 drivers/video/rockchip/rga3/rga_mm.c diff --git a/drivers/video/rockchip/rga3/Makefile b/drivers/video/rockchip/rga3/Makefile index 67db17f02a3b..2b7d6457275b 100644 --- a/drivers/video/rockchip/rga3/Makefile +++ b/drivers/video/rockchip/rga3/Makefile @@ -2,7 +2,7 @@ ccflags-y += -I$(srctree)/$(src)/include -rga3-y := rga_drv.o rga3_reg_info.o rga_dma_buf.o rga_fence.o rga_job.o rga_hw_config.o rga2_reg_info.o rga2_mmu_info.o rga_policy.o +rga3-y := rga_drv.o rga3_reg_info.o rga_dma_buf.o rga_fence.o rga_job.o rga_hw_config.o rga2_reg_info.o rga2_mmu_info.o rga_policy.o rga_mm.o rga3-$(CONFIG_ROCKCHIP_RGA_DEBUGGER) += rga_debugger.o obj-$(CONFIG_ROCKCHIP_MULTI_RGA) += rga3.o diff --git a/drivers/video/rockchip/rga3/include/rga.h b/drivers/video/rockchip/rga3/include/rga.h index 6e23159103fa..502ea1c94432 100644 --- a/drivers/video/rockchip/rga3/include/rga.h +++ b/drivers/video/rockchip/rga3/include/rga.h @@ -13,6 +13,8 @@ #define RGA_IOC_GET_DRVIER_VERSION RGA_IOR(0x1, struct rga_version_t) #define RGA_IOC_GET_HW_VERSION RGA_IOR(0x2, struct rga_hw_versions_t) +#define RGA_IOC_IMPORT_BUFFER RGA_IOWR(0x3, struct rga_buffer_pool) +#define RGA_IOC_RELEASE_BUFFER RGA_IOW(0x4, struct rga_buffer_pool) #define RGA_BLIT_SYNC 0x5017 #define RGA_BLIT_ASYNC 0x5018 @@ -31,6 +33,8 @@ #define SCALE_DOWN_LARGE 1 #define SCALE_UP_LARGE 1 +#define RGA_BUFFER_POOL_SIZE_MAX 40 + #define RGA3_MAJOR_VERSION_MASK (0xF0000000) #define RGA3_MINOR_VERSION_MASK (0x0FF00000) #define RGA3_SVN_VERSION_MASK (0x000FFFFF) @@ -59,6 +63,12 @@ RGA_MODE_X_MIRROR | \ RGA_MODE_Y_MIRROR) +enum rga_memory_type { + RGA_DMA_BUFFER = 0, + RGA_VIRTUAL_ADDRESS, + RGA_PHYSICAL_ADDRESS +}; + enum rga_scale_up_mode { RGA_SCALE_UP_NONE = 0x0, RGA_SCALE_UP_BIC = 0x1, @@ -170,6 +180,18 @@ struct rga_hw_versions_t { uint32_t size; }; +struct rga_external_buffer { + uint64_t memory; + uint32_t type; + + uint32_t handle; +}; + +struct rga_buffer_pool { + struct rga_external_buffer __user *buffers; + uint32_t size; +}; + struct rga_mmu_info_t { unsigned long src0_base_addr; unsigned long src1_base_addr; diff --git a/drivers/video/rockchip/rga3/include/rga_dma_buf.h b/drivers/video/rockchip/rga3/include/rga_dma_buf.h index ce5f285f7ad4..d7cc766e02f1 100644 --- a/drivers/video/rockchip/rga3/include/rga_dma_buf.h +++ b/drivers/video/rockchip/rga3/include/rga_dma_buf.h @@ -15,13 +15,12 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr, unsigned long v_addr, int format, uint32_t w, uint32_t h, unsigned long *StartAddr, unsigned long *size); -void rga_dma_print_session_info(struct rga_dma_session *session); - -int rga_dma_import_fd(int fd); -int rga_dma_release_fd(int fd); - int rga_dma_buf_get(struct rga_job *job); +int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer, + enum dma_data_direction dir, struct device *rga_dev); +void rga_dma_unmap_fd(struct rga_dma_buffer *rga_dma_buffer); + int rga_dma_get_info(struct rga_job *job); void rga_dma_put_info(struct rga_job *job); diff --git a/drivers/video/rockchip/rga3/include/rga_drv.h b/drivers/video/rockchip/rga3/include/rga_drv.h index a5c6a5f08cde..22aa2d4db63b 100644 --- a/drivers/video/rockchip/rga3/include/rga_drv.h +++ b/drivers/video/rockchip/rga3/include/rga_drv.h @@ -131,6 +131,10 @@ struct rga_iommu_dma_cookie { struct iova_domain iovad; }; +/* + * legacy: Wait for the import process to completely replace the current + * dma_map and remove it + */ struct rga_dma_buffer_t { /* DMABUF information */ struct dma_buf *dma_buf; @@ -160,18 +164,41 @@ struct rga_dma_buffer_t { bool use_viraddr; }; -struct rga_dma_buffer_pool { - int __user *fd; - int size; +struct rga_dma_buffer { + /* DMABUF information */ + struct dma_buf *dma_buf; + struct dma_buf_attachment *attach; + struct sg_table *sgt; + + dma_addr_t iova; + unsigned long size; + void *vmap_ptr; + enum dma_data_direction dir; }; -struct rga_dma_session { - struct mutex lock; - /* cached dma buffer list */ - struct list_head cached_list; +struct rga_internal_buffer { + /* DMA buffer */ + struct rga_dma_buffer dma_buffer; - /* the count of buffer in the cached_list */ - int buffer_count; + /* virtual address */ + uint64_t vir_addr; + + /* physical address */ + uint64_t phy_addr; + + /* cached pagetable. */ + struct sg_table *sgt; + uint64_t pt_size; + + /* memory type. */ + uint32_t type; + + uint32_t handle; + + /* It indicates whether the buffer is cached */ + bool cached; + + struct kref refcount; }; /* @@ -268,7 +295,7 @@ struct rga_drvdata_t { struct delayed_work power_off_work; struct wake_lock wake_lock; - struct rga_dma_session *dma_session; + struct rga_mm *mm; #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER struct rga_debugger *debugger; diff --git a/drivers/video/rockchip/rga3/include/rga_mm.h b/drivers/video/rockchip/rga3/include/rga_mm.h new file mode 100644 index 000000000000..55ac7e2c2708 --- /dev/null +++ b/drivers/video/rockchip/rga3/include/rga_mm.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Rockchip Electronics Co., Ltd. + * + * Author: + * Cerf Yu + */ + +#ifndef __LINUX_RKRGA_MM_H_ +#define __LINUX_RKRGA_MM_H_ + +#include "rga_drv.h" + +struct rga_mm { + struct mutex lock; + + /* + * @memory_idr: + * + * Mapping of memory object handles to object pointers. Used by the GEM + * subsystem. Protected by @memory_lock. + */ + struct idr memory_idr; + + /* the count of buffer in the cached_list */ + int buffer_count; +}; + +struct rga_internal_buffer * + rga_mm_internal_buffer_lookup_handle(struct rga_mm *mm_session, uint32_t handle); +void rga_mm_dump_info(struct rga_mm *session); + +uint32_t rga_mm_import_buffer(struct rga_external_buffer *external_buffer); +int rga_mm_release_buffer(uint32_t handle); + +int rga_mm_init(struct rga_mm **session); +int rga_mm_remove(struct rga_mm **session); + +#endif diff --git a/drivers/video/rockchip/rga3/rga_dma_buf.c b/drivers/video/rockchip/rga3/rga_dma_buf.c index 5136b7a6affc..25231324df3d 100644 --- a/drivers/video/rockchip/rga3/rga_dma_buf.c +++ b/drivers/video/rockchip/rga3/rga_dma_buf.c @@ -743,6 +743,66 @@ static int rga_dma_memory_check(struct rga_dma_buffer_t *rga_dma_buffer, return ret; } +int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer, + enum dma_data_direction dir, struct device *rga_dev) +{ + struct dma_buf *dma_buf = NULL; + struct dma_buf_attachment *attach = NULL; + struct sg_table *sgt = NULL; + int ret = 0; + + dma_buf = dma_buf_get(fd); + if (IS_ERR(dma_buf)) { + pr_err("dma_buf_get fail fd[%d]\n", fd); + ret = -EINVAL; + return ret; + } + + attach = dma_buf_attach(dma_buf, rga_dev); + if (IS_ERR(attach)) { + pr_err("Failed to attach dma_buf\n"); + ret = -EINVAL; + goto err_get_attach; + } + + sgt = dma_buf_map_attachment(attach, dir); + if (IS_ERR(sgt)) { + pr_err("Failed to map src attachment\n"); + ret = -EINVAL; + goto err_get_sgt; + } + + rga_dma_buffer->dma_buf = dma_buf; + rga_dma_buffer->attach = attach; + rga_dma_buffer->sgt = sgt; + rga_dma_buffer->size = sg_dma_len(sgt->sgl); + rga_dma_buffer->dir = dir; + + return ret; + +err_get_sgt: + if (attach) + dma_buf_detach(dma_buf, attach); +err_get_attach: + if (dma_buf) + dma_buf_put(dma_buf); + + return ret; +} + +void rga_dma_unmap_fd(struct rga_dma_buffer *rga_dma_buffer) +{ + if (rga_dma_buffer->attach && rga_dma_buffer->sgt) + dma_buf_unmap_attachment(rga_dma_buffer->attach, + rga_dma_buffer->sgt, + rga_dma_buffer->dir); + + if (rga_dma_buffer->attach) { + dma_buf_detach(rga_dma_buffer->dma_buf, rga_dma_buffer->attach); + dma_buf_put(rga_dma_buffer->dma_buf); + } +} + static int rga_dma_map_buffer(struct dma_buf *dma_buf, struct rga_dma_buffer_t *rga_dma_buffer, enum dma_data_direction dir, struct device *rga_dev) diff --git a/drivers/video/rockchip/rga3/rga_drv.c b/drivers/video/rockchip/rga3/rga_drv.c index fc22a3ba264c..9f389c919841 100644 --- a/drivers/video/rockchip/rga3/rga_drv.c +++ b/drivers/video/rockchip/rga3/rga_drv.c @@ -10,6 +10,7 @@ #include "rga2_reg_info.h" #include "rga3_reg_info.h" #include "rga_dma_buf.h" +#include "rga_mm.h" #include "rga_job.h" #include "rga_fence.h" @@ -100,6 +101,124 @@ int rga_power_disable(struct rga_scheduler_t *rga_scheduler) #endif //CONFIG_ROCKCHIP_FPGA +static long rga_ioctl_import_buffer(unsigned long arg) +{ + int i; + int ret = 0; + struct rga_buffer_pool buffer_pool; + struct rga_external_buffer *external_buffer = NULL; + + if (unlikely(copy_from_user(&buffer_pool, + (struct rga_buffer_pool *)arg, + sizeof(buffer_pool)))) { + pr_err("rga_buffer_pool copy_from_user failed!\n"); + return -EFAULT; + } + + if (buffer_pool.size > RGA_BUFFER_POOL_SIZE_MAX) { + pr_err("Cannot import more than %d buffers at a time!\n", + RGA_BUFFER_POOL_SIZE_MAX); + return -EFBIG; + } + + if (buffer_pool.buffers == NULL) { + pr_err("Import buffers is NULL!\n"); + return -EFAULT; + } + + external_buffer = kmalloc(sizeof(struct rga_external_buffer) * buffer_pool.size, + GFP_KERNEL); + if (external_buffer == NULL) { + pr_err("external buffer list alloc error!\n"); + return -ENOMEM; + } + + if (unlikely(copy_from_user(external_buffer, buffer_pool.buffers, + sizeof(struct rga_external_buffer) * buffer_pool.size))) { + pr_err("rga_buffer_pool external_buffer list copy_from_user failed\n"); + ret = -EFAULT; + + goto err_free_external_buffer; + } + + for (i = 0; i < buffer_pool.size; i++) { + ret = rga_mm_import_buffer(&external_buffer[i]); + if (ret < 0) { + pr_err("buffer[%d] mm import buffer failed!\n", i); + + goto err_free_external_buffer; + } + + external_buffer[i].handle = ret; + } + + if (unlikely(copy_to_user(buffer_pool.buffers, external_buffer, + sizeof(struct rga_external_buffer) * buffer_pool.size))) { + pr_err("rga_buffer_pool external_buffer list copy_to_user failed\n"); + ret = -EFAULT; + + goto err_free_external_buffer; + } + +err_free_external_buffer: + kfree(external_buffer); + return ret; +} + +static long rga_ioctl_release_buffer(unsigned long arg) +{ + int i; + int ret = 0; + struct rga_buffer_pool buffer_pool; + struct rga_external_buffer *external_buffer = NULL; + + if (unlikely(copy_from_user(&buffer_pool, + (struct rga_buffer_pool *)arg, + sizeof(buffer_pool)))) { + pr_err("rga_buffer_pool copy_from_user failed!\n"); + return -EFAULT; + } + + if (buffer_pool.size > RGA_BUFFER_POOL_SIZE_MAX) { + pr_err("Cannot release more than %d buffers at a time!\n", + RGA_BUFFER_POOL_SIZE_MAX); + return -EFBIG; + } + + if (buffer_pool.buffers == NULL) { + pr_err("Release buffers is NULL!\n"); + return -EFAULT; + } + + external_buffer = kmalloc(sizeof(struct rga_external_buffer) * buffer_pool.size, + GFP_KERNEL); + if (external_buffer == NULL) { + pr_err("external buffer list alloc error!\n"); + return -ENOMEM; + } + + if (unlikely(copy_from_user(external_buffer, buffer_pool.buffers, + sizeof(struct rga_external_buffer) * buffer_pool.size))) { + pr_err("rga_buffer_pool external_buffer list copy_from_user failed\n"); + ret = -EFAULT; + + goto err_free_external_buffer; + } + + for (i = 0; i < buffer_pool.size; i++) { + ret = rga_mm_release_buffer(external_buffer[i].handle); + if (ret < 0) { + pr_err("buffer[%d] mm release buffer failed!\n", i); + + goto err_free_external_buffer; + } + } + +err_free_external_buffer: + kfree(external_buffer); + return ret; +} + static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) { struct rga_drvdata_t *rga = rga_drvdata; @@ -221,6 +340,16 @@ static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) break; + case RGA_IOC_IMPORT_BUFFER: + ret = rga_ioctl_import_buffer(arg); + + break; + + case RGA_IOC_RELEASE_BUFFER: + ret = rga_ioctl_release_buffer(arg); + + break; + case RGA_IMPORT_DMA: case RGA_RELEASE_DMA: default: @@ -765,6 +894,8 @@ static int __init rga_init(void) return ret; } + rga_mm_init(&rga_drvdata->mm); + #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER rga_debugger_init(&rga_drvdata->debugger); #endif @@ -784,6 +915,8 @@ static void __exit rga_exit(void) rga_debugger_remove(&rga_drvdata->debugger); #endif + rga_mm_remove(&rga_drvdata->mm); + wake_lock_destroy(&rga_drvdata->wake_lock); rga_fence_context_free(rga_drvdata->fence_ctx); diff --git a/drivers/video/rockchip/rga3/rga_mm.c b/drivers/video/rockchip/rga3/rga_mm.c new file mode 100644 index 000000000000..7ca36b4040f9 --- /dev/null +++ b/drivers/video/rockchip/rga3/rga_mm.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Rockchip Electronics Co., Ltd. + * + * Author: Cerf Yu + */ + +#define pr_fmt(fmt) "rga_mm: " fmt + +#include "rga.h" +#include "rga_mm.h" +#include "rga_dma_buf.h" + +static void rga_mm_kref_release_buffer(struct kref *ref) +{ + struct rga_internal_buffer *internal_buffer; + + internal_buffer = container_of(ref, struct rga_internal_buffer, refcount); + + switch (internal_buffer->type) { + case RGA_DMA_BUFFER: + rga_dma_unmap_fd(&internal_buffer->dma_buffer); + break; + case RGA_VIRTUAL_ADDRESS: + // TODO + case RGA_PHYSICAL_ADDRESS: + // TODO + default: + pr_err("Illegal external buffer!\n"); + return; + } + + idr_remove(&rga_drvdata->mm->memory_idr, internal_buffer->handle); + kfree(internal_buffer); + rga_drvdata->mm->buffer_count--; +} + +/* + * Called at driver close to release the memory's handle references. + */ +static int rga_mm_handle_remove(int id, void *ptr, void *data) +{ + struct rga_internal_buffer *internal_buffer = ptr; + + rga_mm_kref_release_buffer(&internal_buffer->refcount); + + return 0; +} + +static struct rga_internal_buffer * +rga_mm_internal_buffer_lookup_external(struct rga_mm *mm_session, + struct rga_external_buffer *external_buffer) +{ + int id; + struct dma_buf *dma_buf = NULL; + struct rga_internal_buffer *temp_buffer = NULL; + struct rga_internal_buffer *output_buffer = NULL; + + WARN_ON(!mutex_is_locked(&mm_session->lock)); + + switch (external_buffer->type) { + case RGA_DMA_BUFFER: + dma_buf = dma_buf_get((int)external_buffer->memory); + if (IS_ERR(dma_buf)) + return (struct rga_internal_buffer *)dma_buf; + + idr_for_each_entry(&mm_session->memory_idr, temp_buffer, id) { + if (temp_buffer->dma_buffer.dma_buf == dma_buf) { + output_buffer = temp_buffer; + break; + } + } + + dma_buf_put(dma_buf); + break; + case RGA_VIRTUAL_ADDRESS: + // TODO + case RGA_PHYSICAL_ADDRESS: + // TODO + default: + pr_err("Illegal external buffer!\n"); + return NULL; + } + + return output_buffer; +} + +struct rga_internal_buffer * +rga_mm_internal_buffer_lookup_handle(struct rga_mm *mm_session, uint32_t handle) +{ + struct rga_internal_buffer *output_buffer; + + WARN_ON(!mutex_is_locked(&mm_session->lock)); + + output_buffer = idr_find(&mm_session->memory_idr, handle); + + return output_buffer; +} + +void rga_mm_dump_info(struct rga_mm *mm_session) +{ + int id; + struct rga_internal_buffer *temp_buffer; + + WARN_ON(!mutex_is_locked(&mm_session->lock)); + + pr_info("rga mm info:\n"); + pr_info("buffer count = %d\n", mm_session->buffer_count); + + idr_for_each_entry(&mm_session->memory_idr, temp_buffer, id) { + pr_info("ID[%d] dma_buf = %p, handle = %d, refcount = %d, cached = %d\n", + id, temp_buffer->dma_buffer.dma_buf, temp_buffer->handle, + kref_read(&temp_buffer->refcount), temp_buffer->cached); + } +} + +uint32_t rga_mm_import_buffer(struct rga_external_buffer *external_buffer) +{ + int ret = 0; + struct rga_mm *mm; + struct rga_internal_buffer *internal_buffer; + + mm = rga_drvdata->mm; + if (mm == NULL) { + pr_err("rga mm is null!\n"); + return -EFAULT; + } + + mutex_lock(&mm->lock); + + /* first, Check whether to rga_mm */ + internal_buffer = rga_mm_internal_buffer_lookup_external(mm, external_buffer); + if (!IS_ERR_OR_NULL(internal_buffer)) { + kref_get(&internal_buffer->refcount); + + mutex_unlock(&mm->lock); + return internal_buffer->handle; + } + + /* finally, map and cached external_buffer in rga_mm */ + internal_buffer = kzalloc(sizeof(struct rga_internal_buffer), GFP_KERNEL); + if (internal_buffer == NULL) { + pr_err("%s alloc internal_buffer error!\n", __func__); + + mutex_unlock(&mm->lock); + return -ENOMEM; + } + + switch (external_buffer->type) { + case RGA_DMA_BUFFER: + ret = rga_dma_map_fd((int)external_buffer->memory, + &internal_buffer->dma_buffer, + DMA_BIDIRECTIONAL, + rga_drvdata->rga_scheduler[0]->dev); + if (ret < 0) { + pr_err("%s map dma buffer error!\n", __func__); + goto FREE_INTERNAL_BUFFER; + } + + internal_buffer->cached = true; + internal_buffer->type = RGA_DMA_BUFFER; + break; + case RGA_VIRTUAL_ADDRESS: + // TODO + case RGA_PHYSICAL_ADDRESS: + // TODO + default: + pr_err("Illegal external buffer!\n"); + ret = -EFAULT; + goto FREE_INTERNAL_BUFFER; + } + + kref_init(&internal_buffer->refcount); + + /* + * Get the user-visible handle using idr. Preload and perform + * allocation under our spinlock. + */ + idr_preload(GFP_KERNEL); + internal_buffer->handle = idr_alloc(&mm->memory_idr, internal_buffer, 1, 0, GFP_KERNEL); + idr_preload_end(); + + mm->buffer_count++; + + mutex_unlock(&mm->lock); + return internal_buffer->handle; + +FREE_INTERNAL_BUFFER: + mutex_unlock(&mm->lock); + kfree(internal_buffer); + + return ret; +} + +int rga_mm_release_buffer(uint32_t handle) +{ + struct rga_mm *mm; + struct rga_internal_buffer *internal_buffer; + + mm = rga_drvdata->mm; + if (mm == NULL) { + pr_err("rga mm is null!\n"); + return -EFAULT; + } + + mutex_lock(&mm->lock); + + /* Find the buffer that has been imported */ + internal_buffer = rga_mm_internal_buffer_lookup_handle(mm, handle); + if (IS_ERR_OR_NULL(internal_buffer)) { + pr_err("This is not a buffer that has been imported, handle = %d\n", (int)handle); + + mutex_unlock(&mm->lock); + return -ENOENT; + } + + kref_put(&internal_buffer->refcount, rga_mm_kref_release_buffer); + + mutex_unlock(&mm->lock); + return 0; +} + +int rga_mm_init(struct rga_mm **mm_session) +{ + struct rga_mm *mm = NULL; + + *mm_session = kzalloc(sizeof(struct rga_mm), GFP_KERNEL); + if (*mm_session == NULL) { + pr_err("can not kzalloc for rga buffer mm_session\n"); + return -ENOMEM; + } + + mm = *mm_session; + + mutex_init(&mm->lock); + idr_init_base(&mm->memory_idr, 1); + + return 0; +} + +int rga_mm_remove(struct rga_mm **mm_session) +{ + struct rga_mm *mm = *mm_session; + + mutex_lock(&mm->lock); + + idr_for_each(&mm->memory_idr, &rga_mm_handle_remove, mm); + idr_destroy(&mm->memory_idr); + + mutex_unlock(&mm->lock); + + kfree(*mm_session); + *mm_session = NULL; + + return 0; +}