video: rockchip: rga3: Add rga_mm.c/rga_mm.h.

1. Add improt/release dma-buf interface.
2. Switch to using idr to manage import/release buffer.

Signed-off-by: Yu Qiaowei <cerf.yu@rock-chips.com>
Change-Id: I40930f3144037435a7c1b78affba2caf81075882

drivers/video/rockchip/rga3/Makefile

@@ -2,7 +2,7 @@
 
 ccflags-y += -I$(srctree)/$(src)/include
 
-rga3-y	:= rga_drv.o rga3_reg_info.o rga_dma_buf.o rga_fence.o rga_job.o rga_hw_config.o rga2_reg_info.o rga2_mmu_info.o rga_policy.o
+rga3-y	:= rga_drv.o rga3_reg_info.o rga_dma_buf.o rga_fence.o rga_job.o rga_hw_config.o rga2_reg_info.o rga2_mmu_info.o rga_policy.o rga_mm.o
 rga3-$(CONFIG_ROCKCHIP_RGA_DEBUGGER) += rga_debugger.o
 
 obj-$(CONFIG_ROCKCHIP_MULTI_RGA)	+= rga3.o

drivers/video/rockchip/rga3/include/rga.h

@@ -13,6 +13,8 @@
 
 #define RGA_IOC_GET_DRVIER_VERSION	RGA_IOR(0x1, struct rga_version_t)
 #define RGA_IOC_GET_HW_VERSION		RGA_IOR(0x2, struct rga_hw_versions_t)
+#define RGA_IOC_IMPORT_BUFFER		RGA_IOWR(0x3, struct rga_buffer_pool)
+#define RGA_IOC_RELEASE_BUFFER		RGA_IOW(0x4, struct rga_buffer_pool)
 
 #define RGA_BLIT_SYNC			0x5017
 #define RGA_BLIT_ASYNC			0x5018
@@ -31,6 +33,8 @@
 #define SCALE_DOWN_LARGE		1
 #define SCALE_UP_LARGE			1
 
+#define RGA_BUFFER_POOL_SIZE_MAX 40
+
 #define RGA3_MAJOR_VERSION_MASK	 (0xF0000000)
 #define RGA3_MINOR_VERSION_MASK	 (0x0FF00000)
 #define RGA3_SVN_VERSION_MASK	 (0x000FFFFF)
@@ -59,6 +63,12 @@
 		RGA_MODE_X_MIRROR | \
 		RGA_MODE_Y_MIRROR)
 
+enum rga_memory_type {
+	RGA_DMA_BUFFER = 0,
+	RGA_VIRTUAL_ADDRESS,
+	RGA_PHYSICAL_ADDRESS
+};
+
 enum rga_scale_up_mode {
 	RGA_SCALE_UP_NONE	= 0x0,
 	RGA_SCALE_UP_BIC	= 0x1,
@@ -170,6 +180,18 @@ struct rga_hw_versions_t {
 	uint32_t size;
 };
 
+struct rga_external_buffer {
+	uint64_t memory;
+	uint32_t type;
+
+	uint32_t handle;
+};
+
+struct rga_buffer_pool {
+	struct rga_external_buffer __user *buffers;
+	uint32_t size;
+};
+
 struct rga_mmu_info_t {
 	unsigned long src0_base_addr;
 	unsigned long src1_base_addr;

drivers/video/rockchip/rga3/include/rga_dma_buf.h

@@ -15,13 +15,12 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr,
 		      unsigned long v_addr, int format, uint32_t w,
 		      uint32_t h, unsigned long *StartAddr, unsigned long *size);
 
-void rga_dma_print_session_info(struct rga_dma_session *session);
-
-int rga_dma_import_fd(int fd);
-int rga_dma_release_fd(int fd);
-
 int rga_dma_buf_get(struct rga_job *job);
 
+int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
+		   enum dma_data_direction dir, struct device *rga_dev);
+void rga_dma_unmap_fd(struct rga_dma_buffer *rga_dma_buffer);
+
 int rga_dma_get_info(struct rga_job *job);
 void rga_dma_put_info(struct rga_job *job);
 

drivers/video/rockchip/rga3/include/rga_drv.h

@@ -131,6 +131,10 @@ struct rga_iommu_dma_cookie {
 	struct iova_domain  iovad;
 };
 
+/*
+ * legacy: Wait for the import process to completely replace the current
+ * dma_map and remove it
+ */
 struct rga_dma_buffer_t {
 	/* DMABUF information */
 	struct dma_buf *dma_buf;
@@ -160,18 +164,41 @@ struct rga_dma_buffer_t {
 	bool use_viraddr;
 };
 
-struct rga_dma_buffer_pool {
-	int __user *fd;
-	int size;
+struct rga_dma_buffer {
+	/* DMABUF information */
+	struct dma_buf *dma_buf;
+	struct dma_buf_attachment *attach;
+	struct sg_table *sgt;
+
+	dma_addr_t iova;
+	unsigned long size;
+	void *vmap_ptr;
+	enum dma_data_direction dir;
 };
 
-struct rga_dma_session {
-	struct mutex lock;
-	/* cached dma buffer list */
-	struct list_head cached_list;
+struct rga_internal_buffer {
+	/* DMA buffer */
+	struct rga_dma_buffer dma_buffer;
 
-	/* the count of buffer in the cached_list */
-	int buffer_count;
+	/* virtual address */
+	uint64_t vir_addr;
+
+	/* physical address */
+	uint64_t phy_addr;
+
+	/* cached pagetable. */
+	struct sg_table *sgt;
+	uint64_t pt_size;
+
+	/* memory type. */
+	uint32_t type;
+
+	uint32_t handle;
+
+	/* It indicates whether the buffer is cached */
+	bool cached;
+
+	struct kref refcount;
 };
 
 /*
@@ -268,7 +295,7 @@ struct rga_drvdata_t {
 	struct delayed_work power_off_work;
 	struct wake_lock wake_lock;
 
-	struct rga_dma_session *dma_session;
+	struct rga_mm *mm;
 
 #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER
 	struct rga_debugger *debugger;

drivers/video/rockchip/rga3/include/rga_mm.h

@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Rockchip Electronics Co., Ltd.
+ *
+ * Author:
+ *  Cerf Yu <cerf.yu@rock-chips.com>
+ */
+
+#ifndef __LINUX_RKRGA_MM_H_
+#define __LINUX_RKRGA_MM_H_
+
+#include "rga_drv.h"
+
+struct rga_mm {
+	struct mutex lock;
+
+	/*
+	 * @memory_idr:
+	 *
+	 * Mapping of memory object handles to object pointers. Used by the GEM
+	 * subsystem. Protected by @memory_lock.
+	 */
+	struct idr memory_idr;
+
+	/* the count of buffer in the cached_list */
+	int buffer_count;
+};
+
+struct rga_internal_buffer *
+	rga_mm_internal_buffer_lookup_handle(struct rga_mm *mm_session, uint32_t handle);
+void rga_mm_dump_info(struct rga_mm *session);
+
+uint32_t rga_mm_import_buffer(struct rga_external_buffer *external_buffer);
+int rga_mm_release_buffer(uint32_t handle);
+
+int rga_mm_init(struct rga_mm **session);
+int rga_mm_remove(struct rga_mm **session);
+
+#endif

drivers/video/rockchip/rga3/rga_dma_buf.c

@@ -743,6 +743,66 @@ static int rga_dma_memory_check(struct rga_dma_buffer_t *rga_dma_buffer,
 	return ret;
 }
 
+int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
+		   enum dma_data_direction dir, struct device *rga_dev)
+{
+	struct dma_buf *dma_buf = NULL;
+	struct dma_buf_attachment *attach = NULL;
+	struct sg_table *sgt = NULL;
+	int ret = 0;
+
+	dma_buf = dma_buf_get(fd);
+	if (IS_ERR(dma_buf)) {
+		pr_err("dma_buf_get fail fd[%d]\n", fd);
+		ret = -EINVAL;
+		return ret;
+	}
+
+	attach = dma_buf_attach(dma_buf, rga_dev);
+	if (IS_ERR(attach)) {
+		pr_err("Failed to attach dma_buf\n");
+		ret = -EINVAL;
+		goto err_get_attach;
+	}
+
+	sgt = dma_buf_map_attachment(attach, dir);
+	if (IS_ERR(sgt)) {
+		pr_err("Failed to map src attachment\n");
+		ret = -EINVAL;
+		goto err_get_sgt;
+	}
+
+	rga_dma_buffer->dma_buf = dma_buf;
+	rga_dma_buffer->attach = attach;
+	rga_dma_buffer->sgt = sgt;
+	rga_dma_buffer->size = sg_dma_len(sgt->sgl);
+	rga_dma_buffer->dir = dir;
+
+	return ret;
+
+err_get_sgt:
+	if (attach)
+		dma_buf_detach(dma_buf, attach);
+err_get_attach:
+	if (dma_buf)
+		dma_buf_put(dma_buf);
+
+	return ret;
+}
+
+void rga_dma_unmap_fd(struct rga_dma_buffer *rga_dma_buffer)
+{
+	if (rga_dma_buffer->attach && rga_dma_buffer->sgt)
+		dma_buf_unmap_attachment(rga_dma_buffer->attach,
+					 rga_dma_buffer->sgt,
+					 rga_dma_buffer->dir);
+
+	if (rga_dma_buffer->attach) {
+		dma_buf_detach(rga_dma_buffer->dma_buf, rga_dma_buffer->attach);
+		dma_buf_put(rga_dma_buffer->dma_buf);
+	}
+}
+
 static int rga_dma_map_buffer(struct dma_buf *dma_buf,
 				 struct rga_dma_buffer_t *rga_dma_buffer,
 				 enum dma_data_direction dir, struct device *rga_dev)

drivers/video/rockchip/rga3/rga_drv.c

@@ -10,6 +10,7 @@
 #include "rga2_reg_info.h"
 #include "rga3_reg_info.h"
 #include "rga_dma_buf.h"
+#include "rga_mm.h"
 
 #include "rga_job.h"
 #include "rga_fence.h"
@@ -100,6 +101,124 @@ int rga_power_disable(struct rga_scheduler_t *rga_scheduler)
 
 #endif //CONFIG_ROCKCHIP_FPGA
 
+static long rga_ioctl_import_buffer(unsigned long arg)
+{
+	int i;
+	int ret = 0;
+	struct rga_buffer_pool buffer_pool;
+	struct rga_external_buffer *external_buffer = NULL;
+
+	if (unlikely(copy_from_user(&buffer_pool,
+				    (struct rga_buffer_pool *)arg,
+				    sizeof(buffer_pool)))) {
+		pr_err("rga_buffer_pool copy_from_user failed!\n");
+		return -EFAULT;
+	}
+
+	if (buffer_pool.size > RGA_BUFFER_POOL_SIZE_MAX) {
+		pr_err("Cannot import more than %d buffers at a time!\n",
+		       RGA_BUFFER_POOL_SIZE_MAX);
+		return -EFBIG;
+	}
+
+	if (buffer_pool.buffers == NULL) {
+		pr_err("Import buffers is NULL!\n");
+		return -EFAULT;
+	}
+
+	external_buffer = kmalloc(sizeof(struct rga_external_buffer) * buffer_pool.size,
+				  GFP_KERNEL);
+	if (external_buffer == NULL) {
+		pr_err("external buffer list alloc error!\n");
+		return -ENOMEM;
+	}
+
+	if (unlikely(copy_from_user(external_buffer, buffer_pool.buffers,
+				    sizeof(struct rga_external_buffer) * buffer_pool.size))) {
+		pr_err("rga_buffer_pool external_buffer list copy_from_user failed\n");
+		ret = -EFAULT;
+
+		goto err_free_external_buffer;
+	}
+
+	for (i = 0; i < buffer_pool.size; i++) {
+		ret = rga_mm_import_buffer(&external_buffer[i]);
+		if (ret < 0) {
+			pr_err("buffer[%d] mm import buffer failed!\n", i);
+
+			goto err_free_external_buffer;
+		}
+
+		external_buffer[i].handle = ret;
+	}
+
+	if (unlikely(copy_to_user(buffer_pool.buffers, external_buffer,
+				  sizeof(struct rga_external_buffer) * buffer_pool.size))) {
+		pr_err("rga_buffer_pool external_buffer list copy_to_user failed\n");
+		ret = -EFAULT;
+
+		goto err_free_external_buffer;
+	}
+
+err_free_external_buffer:
+	kfree(external_buffer);
+	return ret;
+}
+
+static long rga_ioctl_release_buffer(unsigned long arg)
+{
+	int i;
+	int ret = 0;
+	struct rga_buffer_pool buffer_pool;
+	struct rga_external_buffer *external_buffer = NULL;
+
+	if (unlikely(copy_from_user(&buffer_pool,
+				    (struct rga_buffer_pool *)arg,
+				    sizeof(buffer_pool)))) {
+		pr_err("rga_buffer_pool  copy_from_user failed!\n");
+		return -EFAULT;
+	}
+
+	if (buffer_pool.size > RGA_BUFFER_POOL_SIZE_MAX) {
+		pr_err("Cannot release more than %d buffers at a time!\n",
+		       RGA_BUFFER_POOL_SIZE_MAX);
+		return -EFBIG;
+	}
+
+	if (buffer_pool.buffers == NULL) {
+		pr_err("Release buffers is NULL!\n");
+		return -EFAULT;
+	}
+
+	external_buffer = kmalloc(sizeof(struct rga_external_buffer) * buffer_pool.size,
+				  GFP_KERNEL);
+	if (external_buffer == NULL) {
+		pr_err("external buffer list alloc error!\n");
+		return -ENOMEM;
+	}
+
+	if (unlikely(copy_from_user(external_buffer, buffer_pool.buffers,
+				    sizeof(struct rga_external_buffer) * buffer_pool.size))) {
+		pr_err("rga_buffer_pool external_buffer list copy_from_user failed\n");
+		ret = -EFAULT;
+
+		goto err_free_external_buffer;
+	}
+
+	for (i = 0; i < buffer_pool.size; i++) {
+		ret = rga_mm_release_buffer(external_buffer[i].handle);
+		if (ret < 0) {
+			pr_err("buffer[%d] mm release buffer failed!\n", i);
+
+			goto err_free_external_buffer;
+		}
+	}
+
+err_free_external_buffer:
+	kfree(external_buffer);
+	return ret;
+}
+
 static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
 {
 	struct rga_drvdata_t *rga = rga_drvdata;
@@ -221,6 +340,16 @@ static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
 
 		break;
 
+	case RGA_IOC_IMPORT_BUFFER:
+		ret = rga_ioctl_import_buffer(arg);
+
+		break;
+
+	case RGA_IOC_RELEASE_BUFFER:
+		ret = rga_ioctl_release_buffer(arg);
+
+		break;
+
 	case RGA_IMPORT_DMA:
 	case RGA_RELEASE_DMA:
 	default:
@@ -765,6 +894,8 @@ static int __init rga_init(void)
 		return ret;
 	}
 
+	rga_mm_init(&rga_drvdata->mm);
+
 #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER
 	rga_debugger_init(&rga_drvdata->debugger);
 #endif
@@ -784,6 +915,8 @@ static void __exit rga_exit(void)
 	rga_debugger_remove(&rga_drvdata->debugger);
 #endif
 
+	rga_mm_remove(&rga_drvdata->mm);
+
 	wake_lock_destroy(&rga_drvdata->wake_lock);
 
 	rga_fence_context_free(rga_drvdata->fence_ctx);

drivers/video/rockchip/rga3/rga_mm.c

@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Rockchip Electronics Co., Ltd.
+ *
+ * Author: Cerf Yu <cerf.yu@rock-chips.com>
+ */
+
+#define pr_fmt(fmt) "rga_mm: " fmt
+
+#include "rga.h"
+#include "rga_mm.h"
+#include "rga_dma_buf.h"
+
+static void rga_mm_kref_release_buffer(struct kref *ref)
+{
+	struct rga_internal_buffer *internal_buffer;
+
+	internal_buffer = container_of(ref, struct rga_internal_buffer, refcount);
+
+	switch (internal_buffer->type) {
+	case RGA_DMA_BUFFER:
+		rga_dma_unmap_fd(&internal_buffer->dma_buffer);
+		break;
+	case RGA_VIRTUAL_ADDRESS:
+		// TODO
+	case RGA_PHYSICAL_ADDRESS:
+		// TODO
+	default:
+		pr_err("Illegal external buffer!\n");
+		return;
+	}
+
+	idr_remove(&rga_drvdata->mm->memory_idr, internal_buffer->handle);
+	kfree(internal_buffer);
+	rga_drvdata->mm->buffer_count--;
+}
+
+/*
+ * Called at driver close to release the memory's handle references.
+ */
+static int rga_mm_handle_remove(int id, void *ptr, void *data)
+{
+	struct rga_internal_buffer *internal_buffer = ptr;
+
+	rga_mm_kref_release_buffer(&internal_buffer->refcount);
+
+	return 0;
+}
+
+static struct rga_internal_buffer *
+rga_mm_internal_buffer_lookup_external(struct rga_mm *mm_session,
+				       struct rga_external_buffer *external_buffer)
+{
+	int id;
+	struct dma_buf *dma_buf = NULL;
+	struct rga_internal_buffer *temp_buffer = NULL;
+	struct rga_internal_buffer *output_buffer = NULL;
+
+	WARN_ON(!mutex_is_locked(&mm_session->lock));
+
+	switch (external_buffer->type) {
+	case RGA_DMA_BUFFER:
+		dma_buf = dma_buf_get((int)external_buffer->memory);
+		if (IS_ERR(dma_buf))
+			return (struct rga_internal_buffer *)dma_buf;
+
+		idr_for_each_entry(&mm_session->memory_idr, temp_buffer, id) {
+			if (temp_buffer->dma_buffer.dma_buf == dma_buf) {
+				output_buffer = temp_buffer;
+				break;
+			}
+		}
+
+		dma_buf_put(dma_buf);
+		break;
+	case RGA_VIRTUAL_ADDRESS:
+		// TODO
+	case RGA_PHYSICAL_ADDRESS:
+		// TODO
+	default:
+		pr_err("Illegal external buffer!\n");
+		return NULL;
+	}
+
+	return output_buffer;
+}
+
+struct rga_internal_buffer *
+rga_mm_internal_buffer_lookup_handle(struct rga_mm *mm_session, uint32_t handle)
+{
+	struct rga_internal_buffer *output_buffer;
+
+	WARN_ON(!mutex_is_locked(&mm_session->lock));
+
+	output_buffer = idr_find(&mm_session->memory_idr, handle);
+
+	return output_buffer;
+}
+
+void rga_mm_dump_info(struct rga_mm *mm_session)
+{
+	int id;
+	struct rga_internal_buffer *temp_buffer;
+
+	WARN_ON(!mutex_is_locked(&mm_session->lock));
+
+	pr_info("rga mm info:\n");
+	pr_info("buffer count = %d\n", mm_session->buffer_count);
+
+	idr_for_each_entry(&mm_session->memory_idr, temp_buffer, id) {
+		pr_info("ID[%d] dma_buf = %p, handle = %d, refcount = %d, cached = %d\n",
+			id, temp_buffer->dma_buffer.dma_buf, temp_buffer->handle,
+			kref_read(&temp_buffer->refcount), temp_buffer->cached);
+	}
+}
+
+uint32_t rga_mm_import_buffer(struct rga_external_buffer *external_buffer)
+{
+	int ret = 0;
+	struct rga_mm *mm;
+	struct rga_internal_buffer *internal_buffer;
+
+	mm = rga_drvdata->mm;
+	if (mm == NULL) {
+		pr_err("rga mm is null!\n");
+		return -EFAULT;
+	}
+
+	mutex_lock(&mm->lock);
+
+	/* first, Check whether to rga_mm */
+	internal_buffer = rga_mm_internal_buffer_lookup_external(mm, external_buffer);
+	if (!IS_ERR_OR_NULL(internal_buffer)) {
+		kref_get(&internal_buffer->refcount);
+
+		mutex_unlock(&mm->lock);
+		return internal_buffer->handle;
+	}
+
+	/* finally, map and cached external_buffer in rga_mm */
+	internal_buffer = kzalloc(sizeof(struct rga_internal_buffer), GFP_KERNEL);
+	if (internal_buffer == NULL) {
+		pr_err("%s alloc internal_buffer error!\n", __func__);
+
+		mutex_unlock(&mm->lock);
+		return -ENOMEM;
+	}
+
+	switch (external_buffer->type) {
+	case RGA_DMA_BUFFER:
+		ret = rga_dma_map_fd((int)external_buffer->memory,
+				     &internal_buffer->dma_buffer,
+				     DMA_BIDIRECTIONAL,
+				     rga_drvdata->rga_scheduler[0]->dev);
+		if (ret < 0) {
+			pr_err("%s map dma buffer error!\n", __func__);
+			goto FREE_INTERNAL_BUFFER;
+		}
+
+		internal_buffer->cached = true;
+		internal_buffer->type = RGA_DMA_BUFFER;
+		break;
+	case RGA_VIRTUAL_ADDRESS:
+		// TODO
+	case RGA_PHYSICAL_ADDRESS:
+		// TODO
+	default:
+		pr_err("Illegal external buffer!\n");
+		ret = -EFAULT;
+		goto FREE_INTERNAL_BUFFER;
+	}
+
+	kref_init(&internal_buffer->refcount);
+
+	/*
+	 * Get the user-visible handle using idr. Preload and perform
+	 * allocation under our spinlock.
+	 */
+	idr_preload(GFP_KERNEL);
+	internal_buffer->handle = idr_alloc(&mm->memory_idr, internal_buffer, 1, 0, GFP_KERNEL);
+	idr_preload_end();
+
+	mm->buffer_count++;
+
+	mutex_unlock(&mm->lock);
+	return internal_buffer->handle;
+
+FREE_INTERNAL_BUFFER:
+	mutex_unlock(&mm->lock);
+	kfree(internal_buffer);
+
+	return ret;
+}
+
+int rga_mm_release_buffer(uint32_t handle)
+{
+	struct rga_mm *mm;
+	struct rga_internal_buffer *internal_buffer;
+
+	mm = rga_drvdata->mm;
+	if (mm == NULL) {
+		pr_err("rga mm is null!\n");
+		return -EFAULT;
+	}
+
+	mutex_lock(&mm->lock);
+
+	/* Find the buffer that has been imported */
+	internal_buffer = rga_mm_internal_buffer_lookup_handle(mm, handle);
+	if (IS_ERR_OR_NULL(internal_buffer)) {
+		pr_err("This is not a buffer that has been imported, handle = %d\n", (int)handle);
+
+		mutex_unlock(&mm->lock);
+		return -ENOENT;
+	}
+
+	kref_put(&internal_buffer->refcount, rga_mm_kref_release_buffer);
+
+	mutex_unlock(&mm->lock);
+	return 0;
+}
+
+int rga_mm_init(struct rga_mm **mm_session)
+{
+	struct rga_mm *mm = NULL;
+
+	*mm_session = kzalloc(sizeof(struct rga_mm), GFP_KERNEL);
+	if (*mm_session == NULL) {
+		pr_err("can not kzalloc for rga buffer mm_session\n");
+		return -ENOMEM;
+	}
+
+	mm = *mm_session;
+
+	mutex_init(&mm->lock);
+	idr_init_base(&mm->memory_idr, 1);
+
+	return 0;
+}
+
+int rga_mm_remove(struct rga_mm **mm_session)
+{
+	struct rga_mm *mm = *mm_session;
+
+	mutex_lock(&mm->lock);
+
+	idr_for_each(&mm->memory_idr, &rga_mm_handle_remove, mm);
+	idr_destroy(&mm->memory_idr);
+
+	mutex_unlock(&mm->lock);
+
+	kfree(*mm_session);
+	*mm_session = NULL;
+
+	return 0;
+}