driver: rknpu: Add rknpu driver for rk356x, version: 0.5.0

Signed-off-by: Felix Zeng <felix.zeng@rock-chips.com>
Change-Id: Ia9c19e37024d085010ef4c86a420ed5a9c831915

drivers/Kconfig

@@ -239,4 +239,7 @@ source "drivers/most/Kconfig"
 source "drivers/rkflash/Kconfig"
 
 source "drivers/rk_nand/Kconfig"
+
+source "drivers/rknpu/Kconfig"
+
 endmenu

drivers/Makefile

@@ -191,3 +191,4 @@ obj-$(CONFIG_COUNTER)		+= counter/
 obj-$(CONFIG_MOST)		+= most/
 obj-$(CONFIG_RK_FLASH)		+= rkflash/
 obj-$(CONFIG_RK_NAND)		+= rk_nand/
+obj-$(CONFIG_ROCKCHIP_RKNPU)	+= rknpu/

drivers/rknpu/Kconfig

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+menu "RKNPU"
+	depends on ARCH_ROCKCHIP && DRM
+
+config ROCKCHIP_RKNPU
+	tristate "ROCKCHIP_RKNPU"
+	help
+	  rknpu module.
+
+endmenu

drivers/rknpu/Makefile

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_ROCKCHIP_RKNPU) += rknpu.o
+
+ccflags-y += -I$(src)/include
+ccflags-y += -Werror
+
+rknpu-y += rknpu_drv.o
+rknpu-y += rknpu_reset.o
+rknpu-y += rknpu_job.o
+rknpu-y += rknpu_gem.o
+rknpu-y += rknpu_fence.o

drivers/rknpu/include/rknpu_drv.h

@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#ifndef __LINUX_RKNPU_DRV_H_
+#define __LINUX_RKNPU_DRV_H_
+
+#include <linux/completion.h>
+#include <linux/device.h>
+#include <linux/kref.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/regulator/consumer.h>
+#include <linux/version.h>
+
+#include "rknpu_job.h"
+#include "rknpu_fence.h"
+
+#define DRIVER_NAME "rknpu"
+#define DRIVER_DESC "RKNPU driver"
+#define DRIVER_DATE "20210824"
+#define DRIVER_MAJOR 0
+#define DRIVER_MINOR 5
+#define DRIVER_PATCHLEVEL 0
+
+#define LOG_TAG "RKNPU"
+
+#define LOG_INFO(fmt, args...) pr_info(LOG_TAG ": " fmt, ##args)
+#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE
+#define LOG_WARN(fmt, args...) pr_warn(LOG_TAG ": " fmt, ##args)
+#else
+#define LOG_WARN(fmt, args...) pr_warning(LOG_TAG ": " fmt, ##args)
+#endif
+#define LOG_DEBUG(fmt, args...) DRM_DEBUG_DRIVER(LOG_TAG ": " fmt, ##args)
+#define LOG_ERROR(fmt, args...) pr_err(LOG_TAG ": " fmt, ##args)
+
+#define LOG_DEV_INFO(dev, fmt, args...) dev_info(dev, LOG_TAG ": " fmt, ##args)
+#define LOG_DEV_WARN(dev, fmt, args...) dev_warn(dev, LOG_TAG ": " fmt, ##args)
+#define LOG_DEV_DEBUG(dev, fmt, args...)                                       \
+	DRM_DEV_DEBUG_DRIVER(dev, LOG_TAG ": " fmt, ##args)
+#define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args)
+
+struct rknpu_config {
+	__u32 bw_priority_addr;
+	__u32 bw_priority_length;
+	__u64 dma_mask;
+};
+
+/**
+ * RKNPU device
+ *
+ * @base: IO mapped base address for device
+ * @dev: Device instance
+ * @drm_dev: DRM device instance
+ */
+struct rknpu_device {
+	void __iomem *base;
+	struct device *dev;
+	struct device *fake_dev;
+	struct drm_device *drm_dev;
+	atomic_t sequence;
+	spinlock_t lock;
+	spinlock_t irq_lock;
+	struct list_head todo_list;
+	wait_queue_head_t job_done_wq;
+	const struct rknpu_config *config;
+	void __iomem *bw_priority_base;
+	struct rknpu_job *job;
+	struct rknpu_fence_context *fence_ctx;
+	bool iommu_en;
+	struct reset_control *srst_a;
+	struct reset_control *srst_h;
+	struct clk_bulk_data *clks;
+	int num_clks;
+	struct regulator *vdd;
+	struct monitor_dev_info *mdev_info;
+	struct ipa_power_model_data *model_data;
+	struct thermal_cooling_device *devfreq_cooling;
+	struct devfreq *devfreq;
+	unsigned long current_freq;
+	unsigned long current_volt;
+	int bypass_irq_handler;
+	int bypass_soft_reset;
+};
+
+#endif /* __LINUX_RKNPU_DRV_H_ */

drivers/rknpu/include/rknpu_fence.h

@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#ifndef __LINUX_RKNPU_FENCE_H_
+#define __LINUX_RKNPU_FENCE_H_
+
+#include "rknpu_job.h"
+
+struct rknpu_fence_context {
+	unsigned int context;
+	unsigned int seqno;
+	spinlock_t spinlock;
+};
+
+struct rknpu_fence_context *rknpu_fence_context_alloc(void);
+
+void rknpu_fence_context_free(struct rknpu_fence_context *fence_ctx);
+
+int rknpu_fence_alloc(struct rknpu_job *job);
+
+int rknpu_fence_get_fd(struct rknpu_job *job);
+
+#endif /* __LINUX_RKNPU_FENCE_H_ */

drivers/rknpu/include/rknpu_gem.h

@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#ifndef __LINUX_RKNPU_GEM_H
+#define __LINUX_RKNPU_GEM_H
+
+#include <linux/mm_types.h>
+#include <linux/version.h>
+
+#include <drm/drm_device.h>
+#include <drm/drm_vma_manager.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_mode.h>
+
+#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE
+#include <drm/drm_mem_util.h>
+#endif
+
+#define to_rknpu_obj(x) container_of(x, struct rknpu_gem_object, base)
+
+/*
+ * rknpu drm buffer structure.
+ *
+ * @base: a gem object.
+ *	- a new handle to this gem object would be created
+ *	by drm_gem_handle_create().
+ * @flags: indicate memory type to allocated buffer and cache attribute.
+ * @size: size requested from user, in bytes and this size is aligned
+ *	in page unit.
+ * @cookie: cookie returned by dma_alloc_attrs
+ * @kv_addr: kernel virtual address to allocated memory region.
+ * @dma_addr: bus address(accessed by dma) to allocated memory region.
+ *	- this address could be physical address without IOMMU and
+ *	device address with IOMMU.
+ * @pages: Array of backing pages.
+ * @sgt: Imported sg_table.
+ *
+ * P.S. this object would be transferred to user as kms_bo.handle so
+ *	user can access the buffer through kms_bo.handle.
+ */
+struct rknpu_gem_object {
+	struct drm_gem_object base;
+	unsigned int flags;
+	unsigned long size;
+	void *cookie;
+	void __iomem *kv_addr;
+	dma_addr_t dma_addr;
+	unsigned long dma_attrs;
+	unsigned long num_pages;
+	struct page **pages;
+	struct sg_table *sgt;
+	struct drm_mm_node mm_node;
+};
+
+/* create a new buffer with gem object */
+struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev,
+						 unsigned int flags,
+						 unsigned long size);
+
+/* destroy a buffer with gem object */
+void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj);
+
+/* request gem object creation and buffer allocation as the size */
+int rknpu_gem_create_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv);
+
+/* get fake-offset of gem object that can be used with mmap. */
+int rknpu_gem_map_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+
+int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv);
+
+/*
+ * get rknpu drm object,
+ * gem object reference count would be increased.
+ */
+static inline void rknpu_gem_object_get(struct drm_gem_object *obj)
+{
+#if KERNEL_VERSION(4, 13, 0) < LINUX_VERSION_CODE
+	drm_gem_object_get(obj);
+#else
+	drm_gem_object_reference(obj);
+#endif
+}
+
+/*
+ * put rknpu drm object acquired from rknpu_gem_object_find() or rknpu_gem_object_get(),
+ * gem object reference count would be decreased.
+ */
+static inline void rknpu_gem_object_put(struct drm_gem_object *obj)
+{
+#if KERNEL_VERSION(5, 9, 0) <= LINUX_VERSION_CODE
+	drm_gem_object_put(obj);
+#elif KERNEL_VERSION(4, 13, 0) < LINUX_VERSION_CODE
+	drm_gem_object_put_unlocked(obj);
+#else
+	drm_gem_object_unreference_unlocked(obj);
+#endif
+}
+
+/*
+ * get rknpu drm object from gem handle, this function could be used for
+ * other drivers such as 2d/3d acceleration drivers.
+ * with this function call, gem object reference count would be increased.
+ */
+static inline struct rknpu_gem_object *
+rknpu_gem_object_find(struct drm_file *filp, unsigned int handle)
+{
+	struct drm_gem_object *obj;
+
+	obj = drm_gem_object_lookup(filp, handle);
+	if (!obj) {
+		// DRM_ERROR("failed to lookup gem object.\n");
+		return NULL;
+	}
+
+	rknpu_gem_object_put(obj);
+
+	return to_rknpu_obj(obj);
+}
+
+/* get buffer information to memory region allocated by gem. */
+int rknpu_gem_get_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv);
+
+/* free gem object. */
+void rknpu_gem_free_object(struct drm_gem_object *obj);
+
+/* create memory region for drm framebuffer. */
+int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev,
+			  struct drm_mode_create_dumb *args);
+
+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
+/* map memory region for drm framebuffer to user space. */
+int rknpu_gem_dumb_map_offset(struct drm_file *file_priv,
+			      struct drm_device *dev, uint32_t handle,
+			      uint64_t *offset);
+#endif
+
+/* page fault handler and mmap fault address(virtual) to physical memory. */
+#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
+vm_fault_t rknpu_gem_fault(struct vm_fault *vmf);
+#elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
+int rknpu_gem_fault(struct vm_fault *vmf);
+#else
+int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+#endif
+
+/* set vm_flags and we can change the vm attribute to other one at here. */
+int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+
+/* low-level interface prime helpers */
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
+struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev,
+					      struct dma_buf *dma_buf);
+#endif
+struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
+struct drm_gem_object *
+rknpu_gem_prime_import_sg_table(struct drm_device *dev,
+				struct dma_buf_attachment *attach,
+				struct sg_table *sgt);
+void *rknpu_gem_prime_vmap(struct drm_gem_object *obj);
+void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+int rknpu_gem_prime_mmap(struct drm_gem_object *obj,
+			 struct vm_area_struct *vma);
+
+int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv);
+
+static inline void *rknpu_gem_alloc_page(size_t nr_pages)
+{
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
+	return kvmalloc_array(nr_pages, sizeof(struct page *),
+			      GFP_KERNEL | __GFP_ZERO);
+#else
+	return drm_calloc_large(nr_pages, sizeof(struct page *));
+#endif
+}
+
+static inline void rknpu_gem_free_page(void *pages)
+{
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
+	kvfree(pages);
+#else
+	drm_free_large(pages);
+#endif
+}
+
+#endif

drivers/rknpu/include/rknpu_ioctl.h

@@ -0,0 +1,279 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#ifndef __LINUX_RKNPU_IOCTL_H
+#define __LINUX_RKNPU_IOCTL_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#if !defined(__KERNEL__)
+#define __user
+#endif
+
+#define RKNPU_PC_DATA_EXTRA_AMOUNT 4
+
+#define RKNPU_OFFSET_VERSION 0x0
+#define RKNPU_OFFSET_PC_OP_EN 0x8
+#define RKNPU_OFFSET_PC_DATA_ADDR 0x10
+#define RKNPU_OFFSET_PC_DATA_AMOUNT 0x14
+#define RKNPU_OFFSET_PC_TASK_CONTROL 0x30
+#define RKNPU_OFFSET_PC_DMA_BASE_ADDR 0x34
+#define RKNPU_OFFSET_PC_TASK_STATUS 0x3c
+
+#define RKNPU_OFFSET_INT_MASK 0x20
+#define RKNPU_OFFSET_INT_CLEAR 0x24
+#define RKNPU_OFFSET_INT_STATUS 0x28
+
+#define RKNPU_OFFSET_CLR_ALL_RW_AMOUNT 0x8010
+#define RKNPU_OFFSET_DT_WR_AMOUNT 0x8034
+#define RKNPU_OFFSET_DT_RD_AMOUNT 0x8038
+#define RKNPU_OFFSET_WT_RD_AMOUNT 0x803c
+
+#define RKNPU_OFFSET_ENABLE_MASK 0xf008
+
+#define RKNPU_INT_CLEAR 0x1ffff
+
+#define RKNPU_STR_HELPER(x) #x
+
+#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL)                 \
+	RKNPU_STR_HELPER(MAJOR)                                                \
+	"." RKNPU_STR_HELPER(MINOR) "." RKNPU_STR_HELPER(PATCHLEVEL)
+#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL)                   \
+	(MAJOR * 10000 + MINOR * 100 + PATCHLEVEL)
+#define RKNPU_GET_DRV_VERSION_MAJOR(CODE) (CODE / 10000)
+#define RKNPU_GET_DRV_VERSION_MINOR(CODE) ((CODE % 10000) / 100)
+#define RKNPU_GET_DRV_VERSION_PATCHLEVEL(CODE) (CODE % 100)
+
+/* memory type definitions. */
+enum e_rknpu_mem_type {
+	/* physically continuous memory and used as default. */
+	RKNPU_MEM_CONTIGUOUS = 0 << 0,
+	/* physically non-continuous memory. */
+	RKNPU_MEM_NON_CONTIGUOUS = 1 << 0,
+	/* non-cacheable mapping and used as default. */
+	RKNPU_MEM_NON_CACHEABLE = 0 << 1,
+	/* cacheable mapping. */
+	RKNPU_MEM_CACHEABLE = 1 << 1,
+	/* write-combine mapping. */
+	RKNPU_MEM_WRITE_COMBINE = 1 << 2,
+	/* dma attr kernel mapping */
+	RKNPU_MEM_KERNEL_MAPPING = 1 << 3,
+	/* iommu mapping */
+	RKNPU_MEM_IOMMU = 1 << 4,
+	/* zero mapping */
+	RKNPU_MEM_ZEROING = 1 << 5,
+	/* allocate secure buffer */
+	RKNPU_MEM_SECURE = 1 << 6,
+	/* allocate from non-dma32 zone */
+	RKNPU_MEM_NON_DMA32 = 1 << 7,
+	RKNPU_MEM_MASK = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_CACHEABLE |
+			 RKNPU_MEM_WRITE_COMBINE | RKNPU_MEM_KERNEL_MAPPING |
+			 RKNPU_MEM_IOMMU | RKNPU_MEM_ZEROING |
+			 RKNPU_MEM_SECURE | RKNPU_MEM_NON_DMA32
+};
+
+/* sync mode definitions. */
+enum e_rknpu_mem_sync_mode {
+	RKNPU_MEM_SYNC_TO_DEVICE = 1 << 0,
+	RKNPU_MEM_SYNC_FROM_DEVICE = 1 << 1,
+	RKNPU_MEM_SYNC_MASK =
+		RKNPU_MEM_SYNC_TO_DEVICE | RKNPU_MEM_SYNC_FROM_DEVICE
+};
+
+/* job mode definitions. */
+enum e_rknpu_job_mode {
+	RKNPU_JOB_SLAVE = 0 << 0,
+	RKNPU_JOB_PC = 1 << 0,
+	RKNPU_JOB_BLOCK = 0 << 1,
+	RKNPU_JOB_NONBLOCK = 1 << 1,
+	RKNPU_JOB_PINGPONG = 1 << 2,
+	RKNPU_JOB_FENCE = 1 << 3,
+	RKNPU_JOB_MASK = RKNPU_JOB_PC | RKNPU_JOB_NONBLOCK |
+			 RKNPU_JOB_PINGPONG | RKNPU_JOB_FENCE
+};
+
+/* action definitions */
+enum e_rknpu_action {
+	RKNPU_GET_HW_VERSION = 0,
+	RKNPU_GET_DRV_VERSION = 1,
+	RKNPU_GET_FREQ = 2,
+	RKNPU_SET_FREQ = 3,
+	RKNPU_GET_VOLT = 4,
+	RKNPU_SET_VOLT = 5,
+	RKNPU_ACT_RESET = 6,
+	RKNPU_GET_BW_PRIORITY = 7,
+	RKNPU_SET_BW_PRIORITY = 8,
+	RKNPU_GET_BW_EXPECT = 9,
+	RKNPU_SET_BW_EXPECT = 10,
+	RKNPU_GET_BW_TW = 11,
+	RKNPU_SET_BW_TW = 12,
+	RKNPU_ACT_CLR_TOTAL_RW_AMOUNT = 13,
+	RKNPU_GET_DT_WR_AMOUNT = 14,
+	RKNPU_GET_DT_RD_AMOUNT = 15,
+	RKNPU_GET_WT_RD_AMOUNT = 16,
+	RKNPU_GET_TOTAL_RW_AMOUNT = 17,
+	RKNPU_GET_IOMMU_EN = 18,
+	RKNPU_SET_PROC_NICE = 19,
+};
+
+/**
+ * User-desired buffer creation information structure.
+ *
+ * @handle: The handle of the created GEM object.
+ * @flags: user request for setting memory type or cache attributes.
+ * @size: user-desired memory allocation size.
+ *	- this size value would be page-aligned internally.
+ * @obj_addr: address of RKNPU memory object.
+ * @dma_addr: dma address that access by rknpu.
+ */
+struct rknpu_mem_create {
+	__u32 handle;
+	__u32 flags;
+	__u64 size;
+	__u64 obj_addr;
+	__u64 dma_addr;
+};
+
+/**
+ * A structure for getting a fake-offset that can be used with mmap.
+ *
+ * @handle: handle of gem object.
+ * @reserved: just padding to be 64-bit aligned.
+ * @offset: a fake-offset of gem object.
+ */
+struct rknpu_mem_map {
+	__u32 handle;
+	__u32 reserved;
+	__u64 offset;
+};
+
+/**
+ * For destroying DMA buffer
+ *
+ * @handle:	handle of the buffer.
+ * @reserved: reserved for padding.
+ */
+struct rknpu_mem_destroy {
+	__u32 handle;
+	__u32 reserved;
+};
+
+/**
+ * For synchronizing DMA buffer
+ *
+ * @flags: user request for setting memory type or cache attributes.
+ * @reserved: reserved for padding.
+ * @obj_addr: address of RKNPU memory object.
+ * @offset: offset in bytes from start address of buffer.
+ * @size: size of memory region.
+ *
+ */
+struct rknpu_mem_sync {
+	__u32 flags;
+	__u32 reserved;
+	__u64 obj_addr;
+	__u64 offset;
+	__u64 size;
+};
+
+/**
+ * struct rknpu_task structure for task information
+ *
+ * @flags: flags for task
+ * @op_idx: operator index
+ * @enable_mask: enable mask
+ * @int_mask: interrupt mask
+ * @int_clear: interrupt clear
+ * @int_status: interrupt status
+ * @regcfg_amount: register config number
+ * @regcfg_offset: offset for register config
+ * @regcmd_data: data for register command
+ *
+ */
+struct rknpu_task {
+	__u32 flags;
+	__u32 op_idx;
+	__u32 enable_mask;
+	__u32 int_mask;
+	__u32 int_clear;
+	__u32 int_status;
+	__u32 regcfg_amount;
+	__u32 regcfg_offset;
+	__u64 regcmd_data;
+} __packed;
+
+/**
+ * struct rknpu_submit structure for job submit
+ *
+ * @flags: flags for job submit
+ * @timeout: submit timeout
+ * @task_start: task start index
+ * @task_number: task number
+ * @task_counter: task counter
+ * @priority: submit priority
+ * @task_obj_addr: address of task object
+ * @regcfg_obj_addr: address of register config object
+ * @user_data: (optional) user data
+ * @sequence: submit sequence
+ * @core_id: core id of rknpu
+ * @fence_fd: dma fence fd
+ *
+ */
+struct rknpu_submit {
+	__u32 flags;
+	__u32 timeout;
+	__u32 task_start;
+	__u32 task_number;
+	__u32 task_counter;
+	__s32 priority;
+	__u64 task_obj_addr;
+	__u64 regcfg_obj_addr;
+	__u64 user_data;
+	__u64 sequence;
+	__u32 core_id;
+	__s32 fence_fd;
+};
+
+/**
+ * struct rknpu_task structure for action (GET, SET or ACT)
+ *
+ * @flags: flags for action
+ * @value: GET or SET value
+ *
+ */
+struct rknpu_action {
+	__u32 flags;
+	__u32 value;
+};
+
+#define RKNPU_ACTION 0x00
+#define RKNPU_SUBMIT 0x01
+#define RKNPU_MEM_CREATE 0x02
+#define RKNPU_MEM_MAP 0x03
+#define RKNPU_MEM_DESTROY 0x04
+#define RKNPU_MEM_SYNC 0x05
+
+#if defined(__arm__) || defined(__aarch64__)
+
+#include <drm/drm.h>
+
+#define DRM_IOCTL_RKNPU_ACTION                                                 \
+	DRM_IOWR(DRM_COMMAND_BASE + RKNPU_ACTION, struct rknpu_action)
+#define DRM_IOCTL_RKNPU_SUBMIT                                                 \
+	DRM_IOWR(DRM_COMMAND_BASE + RKNPU_SUBMIT, struct rknpu_submit)
+#define DRM_IOCTL_RKNPU_MEM_CREATE                                             \
+	DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_CREATE, struct rknpu_mem_create)
+#define DRM_IOCTL_RKNPU_MEM_MAP                                                \
+	DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_MAP, struct rknpu_mem_map)
+#define DRM_IOCTL_RKNPU_MEM_DESTROY                                            \
+	DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_DESTROY, struct rknpu_mem_destroy)
+#define DRM_IOCTL_RKNPU_MEM_SYNC                                               \
+	DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_SYNC, struct rknpu_mem_sync)
+
+#endif
+
+#endif

drivers/rknpu/include/rknpu_job.h

@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#ifndef __LINUX_RKNPU_JOB_H_
+#define __LINUX_RKNPU_JOB_H_
+
+#include <linux/spinlock.h>
+#include <linux/dma-fence.h>
+
+#include <drm/drm_device.h>
+
+#include "rknpu_ioctl.h"
+
+#define RKNPU_JOB_DONE (1 << 0)
+#define RKNPU_JOB_ASYNC (1 << 1)
+
+struct rknpu_job {
+	struct rknpu_device *rknpu_dev;
+	struct list_head head;
+	struct work_struct cleanup_work;
+	unsigned int flags;
+	int ret;
+	struct rknpu_submit *args;
+	bool args_owner;
+	struct rknpu_task *first_task;
+	struct rknpu_task *last_task;
+	uint32_t int_mask;
+	uint32_t int_status;
+	struct dma_fence *fence;
+	spinlock_t fence_lock;
+	ktime_t timestamp;
+};
+
+irqreturn_t rknpu_irq_handler(int irq, void *data);
+
+int rknpu_submit_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv);
+
+int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version);
+
+int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
+			  uint32_t *expect, uint32_t *tw);
+
+int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
+			  uint32_t expect, uint32_t tw);
+
+int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev);
+
+int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
+			uint32_t *dt_rd, uint32_t *wd_rd);
+
+int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount);
+
+#endif /* __LINUX_RKNPU_JOB_H_ */

drivers/rknpu/include/rknpu_reset.h

@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#include <linux/reset.h>
+
+#include "rknpu_drv.h"
+
+int rknpu_reset_get(struct rknpu_device *rknpu_dev);
+
+int rknpu_soft_reset(struct rknpu_device *rknpu_dev);

drivers/rknpu/rknpu_drv.c

@@ -0,0 +1,833 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/fs.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/time.h>
+#include <linux/uaccess.h>
+#include <linux/ktime.h>
+#include <linux/delay.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/clk.h>
+#include <linux/pm_runtime.h>
+#include <linux/devfreq_cooling.h>
+
+#include <drm/drm_device.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_file.h>
+#include <drm/drm_drv.h>
+
+#include <soc/rockchip/rockchip_opp_select.h>
+#include <soc/rockchip/rockchip_system_monitor.h>
+#include <soc/rockchip/rockchip_ipa.h>
+
+#include "rknpu_ioctl.h"
+#include "rknpu_reset.h"
+#include "rknpu_gem.h"
+#include "rknpu_fence.h"
+#include "rknpu_drv.h"
+
+static int bypass_irq_handler;
+module_param(bypass_irq_handler, int, 0644);
+MODULE_PARM_DESC(bypass_irq_handler,
+		 "bypass RKNPU irq handler if set it to 1, disabled by default");
+
+static int bypass_soft_reset;
+module_param(bypass_soft_reset, int, 0644);
+MODULE_PARM_DESC(bypass_soft_reset,
+		 "bypass RKNPU soft reset if set it to 1, disabled by default");
+
+static const struct rknpu_config rk356x_rknpu_config = {
+	.bw_priority_addr = 0xfe180008,
+	.bw_priority_length = 0x10,
+	.dma_mask = DMA_BIT_MASK(32),
+};
+
+/* driver probe and init */
+static const struct of_device_id rknpu_of_match[] = {
+	{
+		.compatible = "rockchip,rknpu",
+		.data = &rk356x_rknpu_config,
+	},
+	{
+		.compatible = "rockchip,rk3568-rknpu",
+		.data = &rk356x_rknpu_config,
+	},
+	{},
+};
+
+static const struct vm_operations_struct rknpu_gem_vm_ops = {
+	.fault = rknpu_gem_fault,
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+static int rknpu_action_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv);
+
+static const struct drm_ioctl_desc rknpu_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(RKNPU_ACTION, rknpu_action_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(RKNPU_SUBMIT, rknpu_submit_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(RKNPU_MEM_CREATE, rknpu_gem_create_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(RKNPU_MEM_MAP, rknpu_gem_map_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(RKNPU_MEM_DESTROY, rknpu_gem_destroy_ioctl,
+			  DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(RKNPU_MEM_SYNC, rknpu_gem_sync_ioctl,
+			  DRM_RENDER_ALLOW),
+};
+
+static const struct file_operations rknpu_drm_driver_fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.mmap = rknpu_gem_mmap,
+	.poll = drm_poll,
+	.read = drm_read,
+	.unlocked_ioctl = drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl = drm_compat_ioctl,
+#endif
+	.release = drm_release,
+	.llseek = noop_llseek,
+};
+
+static struct drm_driver rknpu_drm_driver = {
+#if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE
+	.driver_features = DRIVER_GEM | DRIVER_RENDER,
+#else
+	.driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER,
+#endif
+	.gem_free_object_unlocked = rknpu_gem_free_object,
+	.gem_vm_ops = &rknpu_gem_vm_ops,
+	.dumb_create = rknpu_gem_dumb_create,
+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
+	.dumb_map_offset = rknpu_gem_dumb_map_offset,
+#else
+	.dumb_map_offset = drm_gem_dumb_map_offset,
+#endif
+	.dumb_destroy = drm_gem_dumb_destroy,
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_export = drm_gem_prime_export,
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
+	.gem_prime_import = rknpu_gem_prime_import,
+#else
+	.gem_prime_import = drm_gem_prime_import,
+#endif
+	.gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table,
+	.gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table,
+	.gem_prime_vmap = rknpu_gem_prime_vmap,
+	.gem_prime_vunmap = rknpu_gem_prime_vunmap,
+	.gem_prime_mmap = rknpu_gem_prime_mmap,
+	.ioctls = rknpu_ioctls,
+	.num_ioctls = ARRAY_SIZE(rknpu_ioctls),
+	.fops = &rknpu_drm_driver_fops,
+	.name = DRIVER_NAME,
+	.desc = DRIVER_DESC,
+	.date = DRIVER_DATE,
+	.major = DRIVER_MAJOR,
+	.minor = DRIVER_MINOR,
+	.patchlevel = DRIVER_PATCHLEVEL,
+};
+
+static int rknpu_get_drv_version(uint32_t *version)
+{
+	*version = RKNPU_GET_DRV_VERSION_CODE(DRIVER_MAJOR, DRIVER_MINOR,
+					      DRIVER_PATCHLEVEL);
+	return 0;
+}
+
+static int rknpu_action_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file_priv)
+{
+	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
+	struct rknpu_action *args = data;
+	int ret = -EINVAL;
+
+	switch (args->flags) {
+	case RKNPU_GET_HW_VERSION:
+		ret = rknpu_get_hw_version(rknpu_dev, &args->value);
+		break;
+	case RKNPU_GET_DRV_VERSION:
+		ret = rknpu_get_drv_version(&args->value);
+		break;
+	case RKNPU_GET_FREQ:
+		args->value = rknpu_dev->current_freq;
+		ret = 0;
+		break;
+	case RKNPU_SET_FREQ:
+		break;
+	case RKNPU_GET_VOLT:
+		args->value = rknpu_dev->current_volt;
+		ret = 0;
+		break;
+	case RKNPU_SET_VOLT:
+		break;
+	case RKNPU_ACT_RESET:
+		ret = rknpu_soft_reset(rknpu_dev);
+		break;
+	case RKNPU_GET_BW_PRIORITY:
+		ret = rknpu_get_bw_priority(rknpu_dev, &args->value, NULL,
+					    NULL);
+		break;
+	case RKNPU_SET_BW_PRIORITY:
+		ret = rknpu_set_bw_priority(rknpu_dev, args->value, 0, 0);
+		break;
+	case RKNPU_GET_BW_EXPECT:
+		ret = rknpu_get_bw_priority(rknpu_dev, NULL, &args->value,
+					    NULL);
+		break;
+	case RKNPU_SET_BW_EXPECT:
+		ret = rknpu_set_bw_priority(rknpu_dev, 0, args->value, 0);
+		break;
+	case RKNPU_GET_BW_TW:
+		ret = rknpu_get_bw_priority(rknpu_dev, NULL, NULL,
+					    &args->value);
+		break;
+	case RKNPU_SET_BW_TW:
+		ret = rknpu_set_bw_priority(rknpu_dev, 0, 0, args->value);
+		break;
+	case RKNPU_ACT_CLR_TOTAL_RW_AMOUNT:
+		ret = rknpu_clear_rw_amount(rknpu_dev);
+		break;
+	case RKNPU_GET_DT_WR_AMOUNT:
+		ret = rknpu_get_rw_amount(rknpu_dev, &args->value, NULL, NULL);
+		break;
+	case RKNPU_GET_DT_RD_AMOUNT:
+		ret = rknpu_get_rw_amount(rknpu_dev, NULL, &args->value, NULL);
+		break;
+	case RKNPU_GET_WT_RD_AMOUNT:
+		ret = rknpu_get_rw_amount(rknpu_dev, NULL, NULL, &args->value);
+		break;
+	case RKNPU_GET_TOTAL_RW_AMOUNT:
+		ret = rknpu_get_total_rw_amount(rknpu_dev, &args->value);
+		break;
+	case RKNPU_GET_IOMMU_EN:
+		args->value = rknpu_dev->iommu_en;
+		ret = 0;
+		break;
+	case RKNPU_SET_PROC_NICE:
+		set_user_nice(current, *(int32_t *)&args->value);
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static bool rknpu_is_iommu_enable(struct device *dev)
+{
+	struct device_node *iommu = NULL;
+
+	iommu = of_parse_phandle(dev->of_node, "iommus", 0);
+	if (!iommu) {
+		LOG_DEV_INFO(
+			dev,
+			"rknpu iommu device-tree entry not found!, using non-iommu mode\n");
+		return false;
+	}
+
+	if (!of_device_is_available(iommu)) {
+		LOG_DEV_INFO(dev,
+			     "rknpu iommu is disabled, using non-iommu mode\n");
+		of_node_put(iommu);
+		return false;
+	}
+	of_node_put(iommu);
+
+	LOG_DEV_INFO(dev, "rknpu iommu is enabled, using iommu mode\n");
+
+	return true;
+}
+
+static int drm_fake_dev_register(struct rknpu_device *rknpu_dev)
+{
+	const struct platform_device_info rknpu_dev_info = {
+		.name = "rknpu_dev",
+		.id = PLATFORM_DEVID_AUTO,
+		.dma_mask = rknpu_dev->config->dma_mask,
+	};
+	struct platform_device *pdev = NULL;
+	int ret = -EINVAL;
+
+	pdev = platform_device_register_full(&rknpu_dev_info);
+	if (pdev) {
+		ret = of_dma_configure(&pdev->dev, NULL, true);
+		if (ret) {
+			platform_device_unregister(pdev);
+			pdev = NULL;
+		}
+	}
+
+	rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL;
+
+	return ret;
+}
+
+static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev)
+{
+	struct platform_device *pdev = NULL;
+
+	if (!rknpu_dev->fake_dev)
+		return;
+
+	pdev = to_platform_device(rknpu_dev->fake_dev);
+
+	platform_device_unregister(pdev);
+}
+
+static int rknpu_drm_probe(struct rknpu_device *rknpu_dev)
+{
+	struct device *dev = rknpu_dev->dev;
+	struct drm_device *drm_dev = NULL;
+	int ret = -EINVAL;
+
+	drm_dev = drm_dev_alloc(&rknpu_drm_driver, dev);
+	if (IS_ERR(drm_dev))
+		return PTR_ERR(drm_dev);
+
+	/* register the DRM device */
+	ret = drm_dev_register(drm_dev, 0);
+	if (ret < 0)
+		goto err_free_drm;
+
+	drm_dev->dev_private = rknpu_dev;
+	rknpu_dev->drm_dev = drm_dev;
+
+	drm_fake_dev_register(rknpu_dev);
+
+	return 0;
+
+err_free_drm:
+#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
+	drm_dev_put(drm_dev);
+#else
+	drm_dev_unref(drm_dev);
+#endif
+
+	return ret;
+}
+
+static void rknpu_drm_remove(struct rknpu_device *rknpu_dev)
+{
+	struct drm_device *drm_dev = rknpu_dev->drm_dev;
+
+	drm_fake_dev_unregister(rknpu_dev);
+
+	drm_dev_unregister(drm_dev);
+
+#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
+	drm_dev_put(drm_dev);
+#else
+	drm_dev_unref(drm_dev);
+#endif
+}
+
+static int rknpu_power_on(struct rknpu_device *rknpu_dev)
+{
+	struct device *dev = rknpu_dev->dev;
+	int ret = -EINVAL;
+
+	ret = regulator_enable(rknpu_dev->vdd);
+	if (ret) {
+		LOG_DEV_ERROR(
+			dev, "failed to enable regulator for rknpu, ret = %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = clk_bulk_prepare_enable(rknpu_dev->num_clks, rknpu_dev->clks);
+	if (ret) {
+		LOG_DEV_ERROR(dev, "failed to enable clk for rknpu, ret = %d\n",
+			      ret);
+		return ret;
+	}
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		LOG_DEV_ERROR(dev,
+			      "failed to get pm runtime for rknpu, ret = %d\n",
+			      ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int rknpu_power_off(struct rknpu_device *rknpu_dev)
+{
+	struct device *dev = rknpu_dev->dev;
+
+	pm_runtime_put_sync(dev);
+
+	clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks);
+
+	regulator_disable(rknpu_dev->vdd);
+
+	return 0;
+}
+
+static struct monitor_dev_profile npu_mdevp = {
+	.type = MONITOR_TPYE_DEV,
+	.low_temp_adjust = rockchip_monitor_dev_low_temp_adjust,
+	.high_temp_adjust = rockchip_monitor_dev_high_temp_adjust,
+};
+
+static int npu_devfreq_target(struct device *dev, unsigned long *target_freq,
+			      u32 flags)
+{
+	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
+	struct dev_pm_opp *opp = NULL;
+	unsigned long freq = *target_freq;
+	unsigned long old_freq = rknpu_dev->current_freq;
+	unsigned long volt, old_volt = rknpu_dev->current_volt;
+	int ret = -EINVAL;
+
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+	rcu_read_lock();
+#endif
+
+	opp = devfreq_recommended_opp(dev, &freq, flags);
+	if (IS_ERR(opp)) {
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+		rcu_read_unlock();
+#endif
+		LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp));
+		return PTR_ERR(opp);
+	}
+	volt = dev_pm_opp_get_voltage(opp);
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+	rcu_read_unlock();
+#endif
+
+	/*
+	 * Only update if there is a change of frequency
+	 */
+	if (old_freq == freq) {
+		*target_freq = freq;
+		if (old_volt == volt)
+			return 0;
+		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
+		if (ret) {
+			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
+			return ret;
+		}
+		rknpu_dev->current_volt = volt;
+		return 0;
+	}
+
+	if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) {
+		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
+		if (ret) {
+			LOG_DEV_ERROR(dev, "failed to increase volt %lu\n",
+				      volt);
+			return ret;
+		}
+	}
+	LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt,
+		      freq, volt);
+	ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
+	if (ret) {
+		LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
+		return ret;
+	}
+	*target_freq = freq;
+	rknpu_dev->current_freq = freq;
+
+	if (rknpu_dev->devfreq)
+		rknpu_dev->devfreq->last_status.current_frequency = freq;
+
+	if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) {
+		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
+		if (ret) {
+			LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n",
+				      volt);
+			return ret;
+		}
+	}
+	rknpu_dev->current_volt = volt;
+
+	return ret;
+}
+
+static int npu_devfreq_get_dev_status(struct device *dev,
+				      struct devfreq_dev_status *stat)
+{
+	return 0;
+}
+
+static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
+
+	*freq = rknpu_dev->current_freq;
+
+	return 0;
+}
+
+static struct devfreq_dev_profile npu_devfreq_profile = {
+	.polling_ms = 50,
+	.target = npu_devfreq_target,
+	.get_dev_status = npu_devfreq_get_dev_status,
+	.get_cur_freq = npu_devfreq_get_cur_freq,
+};
+
+static unsigned long npu_get_static_power(struct devfreq *devfreq,
+					  unsigned long voltage)
+{
+	struct device *dev = devfreq->dev.parent;
+	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
+
+	if (!rknpu_dev->model_data)
+		return 0;
+
+	return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage);
+}
+
+static struct devfreq_cooling_power npu_cooling_power = {
+	.get_static_power = &npu_get_static_power,
+};
+
+static int npu_devfreq_adjust_current_freq_volt(struct device *dev,
+						struct rknpu_device *rknpu_dev)
+{
+	unsigned long volt, old_freq, freq;
+	struct dev_pm_opp *opp = NULL;
+	int ret = -EINVAL;
+
+	old_freq = clk_get_rate(rknpu_dev->clks[0].clk);
+	freq = old_freq;
+
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+	rcu_read_lock();
+#endif
+
+	opp = devfreq_recommended_opp(dev, &freq, 0);
+	volt = dev_pm_opp_get_voltage(opp);
+
+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE
+	rcu_read_unlock();
+#endif
+
+	if (freq >= old_freq && rknpu_dev->vdd) {
+		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
+		if (ret) {
+			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
+			return ret;
+		}
+	}
+	LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq,
+		      volt);
+	ret = clk_set_rate(rknpu_dev->clks[0].clk, freq);
+	if (ret) {
+		LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq);
+		return ret;
+	}
+	if (freq < old_freq && rknpu_dev->vdd) {
+		ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX);
+		if (ret) {
+			LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt);
+			return ret;
+		}
+	}
+	rknpu_dev->current_freq = freq;
+	rknpu_dev->current_volt = volt;
+
+	return 0;
+}
+
+static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev)
+{
+	struct device *dev = rknpu_dev->dev;
+	struct devfreq_dev_profile *dp = &npu_devfreq_profile;
+	int ret = -EINVAL;
+
+	ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu");
+
+	if (ret) {
+		LOG_DEV_ERROR(dev, "failed to init_opp_table\n");
+		return ret;
+	}
+
+	ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev);
+	if (ret) {
+		LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n");
+		return ret;
+	}
+	dp->initial_freq = rknpu_dev->current_freq;
+
+	rknpu_dev->devfreq =
+		devm_devfreq_add_device(dev, dp, "userspace", NULL);
+	if (IS_ERR(rknpu_dev->devfreq)) {
+		LOG_DEV_ERROR(dev, "failed to add devfreq\n");
+		return PTR_ERR(rknpu_dev->devfreq);
+	}
+	devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq);
+
+	rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq;
+	rknpu_dev->devfreq->last_status.total_time = 1;
+	rknpu_dev->devfreq->last_status.busy_time = 1;
+
+	npu_mdevp.data = rknpu_dev->devfreq;
+	rknpu_dev->mdev_info =
+		rockchip_system_monitor_register(dev, &npu_mdevp);
+	if (IS_ERR(rknpu_dev->mdev_info)) {
+		LOG_DEV_DEBUG(dev, "without system monitor\n");
+		rknpu_dev->mdev_info = NULL;
+	}
+
+	of_property_read_u32(dev->of_node, "dynamic-power-coefficient",
+			     (u32 *)&npu_cooling_power.dyn_power_coeff);
+
+#if KERNEL_VERSION(4, 4, 179) <= LINUX_VERSION_CODE
+	rknpu_dev->model_data =
+		rockchip_ipa_power_model_init(dev, "npu_leakage");
+	if (IS_ERR_OR_NULL(rknpu_dev->model_data)) {
+#else
+	ret = rockchip_ipa_power_model_init(dev, &rknpu_dev->model_data);
+	if (ret) {
+#endif
+		rknpu_dev->model_data = NULL;
+		LOG_DEV_ERROR(dev, "failed to initialize power model\n");
+	} else if (rknpu_dev->model_data->dynamic_coefficient) {
+		npu_cooling_power.dyn_power_coeff =
+			rknpu_dev->model_data->dynamic_coefficient;
+	}
+
+	if (!npu_cooling_power.dyn_power_coeff) {
+		LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n");
+		goto out;
+	}
+
+#if KERNEL_VERSION(4, 4, 179) > LINUX_VERSION_CODE
+	rockchip_of_get_leakage(dev, "npu_leakage",
+				&rknpu_dev->model_data->leakage);
+#endif
+	rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power(
+		dev->of_node, rknpu_dev->devfreq, &npu_cooling_power);
+	if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling))
+		LOG_DEV_ERROR(dev, "failed to register cooling device\n");
+
+out:
+	return 0;
+}
+
+static int rknpu_probe(struct platform_device *pdev)
+{
+	struct resource *res = NULL;
+	struct rknpu_device *rknpu_dev = NULL;
+	struct device *dev = &pdev->dev;
+	const struct of_device_id *match = NULL;
+	const struct rknpu_config *config = NULL;
+	int ret = -EINVAL;
+
+	if (!pdev->dev.of_node) {
+		LOG_DEV_ERROR(dev, "rknpu device-tree data is missing!\n");
+		return -ENODEV;
+	}
+
+	match = of_match_device(rknpu_of_match, dev);
+	if (!match) {
+		LOG_DEV_ERROR(dev, "rknpu device-tree entry is missing!\n");
+		return -ENODEV;
+	}
+
+	rknpu_dev = devm_kzalloc(dev, sizeof(*rknpu_dev), GFP_KERNEL);
+	if (!rknpu_dev) {
+		LOG_DEV_ERROR(dev, "failed to allocate rknpu device!\n");
+		return -ENOMEM;
+	}
+
+	config = of_device_get_match_data(dev);
+	if (!config)
+		return -EINVAL;
+
+	rknpu_dev->config = config;
+	rknpu_dev->dev = dev;
+
+	rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev);
+	if (!rknpu_dev->iommu_en) {
+		/* Initialize reserved memory resources */
+		ret = of_reserved_mem_device_init(dev);
+		if (!ret) {
+			LOG_DEV_INFO(
+				dev,
+				"initialize reserved memory for rknpu device!\n");
+		}
+	}
+
+	rknpu_dev->bypass_irq_handler = bypass_irq_handler;
+	rknpu_dev->bypass_soft_reset = bypass_soft_reset;
+
+	rknpu_reset_get(rknpu_dev);
+
+	rknpu_dev->num_clks = devm_clk_bulk_get_all(dev, &rknpu_dev->clks);
+	if (rknpu_dev->num_clks < 1) {
+		LOG_DEV_ERROR(dev, "failed to get clk source for rknpu\n");
+		return -ENODEV;
+	}
+
+	rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu");
+	if (IS_ERR(rknpu_dev->vdd)) {
+		if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) {
+			ret = PTR_ERR(rknpu_dev->vdd);
+			LOG_DEV_ERROR(
+				dev,
+				"failed to get vdd regulator for rknpu: %d\n",
+				ret);
+			return ret;
+		}
+		rknpu_dev->vdd = NULL;
+	}
+
+	spin_lock_init(&rknpu_dev->lock);
+	spin_lock_init(&rknpu_dev->irq_lock);
+	INIT_LIST_HEAD(&rknpu_dev->todo_list);
+	init_waitqueue_head(&rknpu_dev->job_done_wq);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		LOG_DEV_ERROR(dev, "failed to get memory resource for rknpu\n");
+		return -ENXIO;
+	}
+
+	rknpu_dev->base = devm_ioremap_resource(dev, res);
+	if (PTR_ERR(rknpu_dev->base) == -EBUSY) {
+		rknpu_dev->base =
+			devm_ioremap(dev, res->start, resource_size(res));
+	}
+
+	if (IS_ERR(rknpu_dev->base)) {
+		LOG_DEV_ERROR(dev, "failed to remap register for rknpu\n");
+		return PTR_ERR(rknpu_dev->base);
+	}
+
+	rknpu_dev->bw_priority_base = devm_ioremap(
+		dev, config->bw_priority_addr, config->bw_priority_length);
+	if (IS_ERR(rknpu_dev->bw_priority_base)) {
+		LOG_DEV_ERROR(
+			rknpu_dev->dev,
+			"failed to remap bw priority register for rknpu\n");
+		rknpu_dev->bw_priority_base = NULL;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	if (!res) {
+		LOG_DEV_ERROR(dev,
+			      "failed to get interrupt resource for rknpu\n");
+		return -ENXIO;
+	}
+
+	if (!rknpu_dev->bypass_irq_handler) {
+		ret = devm_request_irq(dev, res->start, rknpu_irq_handler,
+				       IRQF_SHARED, dev_name(dev), rknpu_dev);
+		if (ret) {
+			LOG_DEV_ERROR(dev, "failed to request irq for rknpu\n");
+			return ret;
+		}
+	} else {
+		LOG_DEV_WARN(dev, "bypass irq handler!\n");
+	}
+
+	ret = rknpu_drm_probe(rknpu_dev);
+	if (ret) {
+		LOG_DEV_ERROR(dev, "failed to probe device for rknpu\n");
+		return ret;
+	}
+
+	rknpu_dev->fence_ctx = rknpu_fence_context_alloc();
+	if (IS_ERR(rknpu_dev->fence_ctx)) {
+		LOG_DEV_ERROR(dev,
+			      "failed to allocate fence context for rknpu\n");
+		ret = PTR_ERR(rknpu_dev->fence_ctx);
+		goto err_remove_drm;
+	}
+
+	platform_set_drvdata(pdev, rknpu_dev);
+
+	rknpu_devfreq_init(rknpu_dev);
+
+	pm_runtime_enable(dev);
+
+	ret = rknpu_power_on(rknpu_dev);
+	if (ret)
+		goto err_free_fence_context;
+
+	return 0;
+
+err_free_fence_context:
+	rknpu_fence_context_free(rknpu_dev->fence_ctx);
+err_remove_drm:
+	rknpu_drm_remove(rknpu_dev);
+
+	return ret;
+}
+
+static int rknpu_remove(struct platform_device *pdev)
+{
+	struct rknpu_device *rknpu_dev = platform_get_drvdata(pdev);
+
+	WARN_ON(rknpu_dev->job);
+	WARN_ON(!list_empty(&rknpu_dev->todo_list));
+
+	rknpu_drm_remove(rknpu_dev);
+
+	rknpu_fence_context_free(rknpu_dev->fence_ctx);
+
+	rknpu_power_off(rknpu_dev);
+
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static struct platform_driver rknpu_driver = {
+	.probe = rknpu_probe,
+	.remove = rknpu_remove,
+	.driver = {
+		.owner = THIS_MODULE,
+		.name = "RKNPU",
+		.of_match_table = of_match_ptr(rknpu_of_match),
+	},
+};
+
+static int rknpu_init(void)
+{
+	return platform_driver_register(&rknpu_driver);
+}
+
+static void rknpu_exit(void)
+{
+	platform_driver_unregister(&rknpu_driver);
+}
+
+late_initcall(rknpu_init);
+module_exit(rknpu_exit);
+
+MODULE_DESCRIPTION("RKNPU driver");
+MODULE_AUTHOR("Felix Zeng <felix.zeng@rock-chips.com>");
+MODULE_ALIAS("rockchip-rknpu");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR,
+					    DRIVER_PATCHLEVEL));

drivers/rknpu/rknpu_fence.c

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#include <linux/slab.h>
+#include <linux/file.h>
+#include <linux/dma-fence.h>
+#include <linux/sync_file.h>
+
+#include "rknpu_drv.h"
+#include "rknpu_job.h"
+
+#include "rknpu_fence.h"
+
+static const char *rknpu_fence_get_name(struct dma_fence *fence)
+{
+	return DRIVER_NAME;
+}
+
+static const struct dma_fence_ops rknpu_fence_ops = {
+	.get_driver_name = rknpu_fence_get_name,
+	.get_timeline_name = rknpu_fence_get_name,
+};
+
+struct rknpu_fence_context *rknpu_fence_context_alloc(void)
+{
+	struct rknpu_fence_context *fence_ctx = NULL;
+
+	fence_ctx = kzalloc(sizeof(*fence_ctx), GFP_KERNEL);
+	if (!fence_ctx)
+		return ERR_PTR(-ENOMEM);
+
+	fence_ctx->context = dma_fence_context_alloc(1);
+	spin_lock_init(&fence_ctx->spinlock);
+
+	return fence_ctx;
+}
+
+void rknpu_fence_context_free(struct rknpu_fence_context *fence_ctx)
+{
+	if (!IS_ERR(fence_ctx))
+		kfree(fence_ctx);
+}
+
+int rknpu_fence_alloc(struct rknpu_job *job)
+{
+	struct rknpu_fence_context *fence_ctx = job->rknpu_dev->fence_ctx;
+	struct dma_fence *fence = NULL;
+
+	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+	if (!fence)
+		return -ENOMEM;
+
+	dma_fence_init(fence, &rknpu_fence_ops, &job->fence_lock,
+		       fence_ctx->context, ++fence_ctx->seqno);
+
+	job->fence = fence;
+
+	return 0;
+}
+
+int rknpu_fence_get_fd(struct rknpu_job *job)
+{
+	struct sync_file *sync_file = NULL;
+	int fence_fd = -1;
+
+	if (!job->fence)
+		return -EINVAL;
+
+	fence_fd = get_unused_fd_flags(O_CLOEXEC);
+	if (fence_fd < 0)
+		return fence_fd;
+
+	sync_file = sync_file_create(job->fence);
+	if (!sync_file)
+		return -ENOMEM;
+
+	fd_install(fence_fd, sync_file->file);
+
+	return fence_fd;
+}

drivers/rknpu/rknpu_gem.c

@@ -0,0 +1,983 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#include <drm/drm_device.h>
+#include <drm/drm_vma_manager.h>
+#include <drm/drm_prime.h>
+
+#include <linux/shmem_fs.h>
+#include <linux/dma-buf.h>
+#include <linux/pfn_t.h>
+#include <linux/version.h>
+
+#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
+#include <linux/dma-map-ops.h>
+#endif
+
+#include "rknpu_drv.h"
+#include "rknpu_ioctl.h"
+#include "rknpu_gem.h"
+
+#define RKNPU_GEM_ALLOC_FROM_PAGES 0
+
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj)
+{
+	struct drm_device *drm = rknpu_obj->base.dev;
+	struct scatterlist *s = NULL;
+	int ret = -EINVAL, i = 0;
+
+	rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base);
+	if (IS_ERR(rknpu_obj->pages))
+		return PTR_ERR(rknpu_obj->pages);
+
+	rknpu_obj->num_pages = rknpu_obj->base.size >> PAGE_SHIFT;
+
+#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
+	rknpu_obj->sgt = drm_prime_pages_to_sg(drm, rknpu_obj->pages,
+					       rknpu_obj->num_pages);
+#else
+	rknpu_obj->sgt =
+		drm_prime_pages_to_sg(rknpu_obj->pages, rknpu_obj->num_pages);
+#endif
+	if (IS_ERR(rknpu_obj->sgt)) {
+		ret = PTR_ERR(rknpu_obj->sgt);
+		goto put_pages;
+	}
+
+	for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
+		sg_dma_address(s) = sg_phys(s);
+		LOG_DEBUG(
+			"gem pages alloc sgt[%d], phys_address: %#llx, length: %#x\n",
+			i, (__u64)s->dma_address, s->length);
+	}
+
+	ret = dma_map_sg_attrs(drm->dev, rknpu_obj->sgt->sgl,
+			       rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL,
+			       rknpu_obj->dma_attrs);
+	if (ret == 0) {
+		LOG_DEV_ERROR(drm->dev, "failed to map sg table.\n");
+		ret = -EFAULT;
+		goto free_sgt;
+	}
+
+	if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
+		rknpu_obj->kv_addr =
+			vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
+			     PAGE_KERNEL);
+	}
+
+	rknpu_obj->dma_addr = (__u64)sg_dma_address(rknpu_obj->sgt->sgl);
+
+	return 0;
+
+free_sgt:
+	sg_free_table(rknpu_obj->sgt);
+	kfree(rknpu_obj->sgt);
+put_pages:
+	drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
+
+	return ret;
+}
+
+static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj)
+{
+	struct drm_device *drm = rknpu_obj->base.dev;
+
+	if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING)
+		vunmap(rknpu_obj->kv_addr);
+
+	dma_map_sg_attrs(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
+			 DMA_BIDIRECTIONAL, rknpu_obj->dma_attrs);
+	drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true);
+	sg_free_table(rknpu_obj->sgt);
+	kfree(rknpu_obj->sgt);
+}
+#endif
+
+static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj)
+{
+	struct drm_device *drm = rknpu_obj->base.dev;
+	struct rknpu_device *rknpu_dev = drm->dev_private;
+	unsigned int nr_pages = 0;
+	struct sg_table *sgt = NULL;
+	struct scatterlist *s = NULL;
+	gfp_t gfp_mask = GFP_KERNEL;
+	int ret = -EINVAL, i = 0;
+
+	if (rknpu_obj->dma_addr) {
+		LOG_DEBUG("buffer already allocated.\n");
+		return 0;
+	}
+
+	rknpu_obj->dma_attrs = 0;
+
+	/*
+	 * if RKNPU_MEM_CONTIGUOUS, fully physically contiguous memory
+	 * region will be allocated else physically contiguous
+	 * as possible.
+	 */
+	if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS))
+		rknpu_obj->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS;
+
+	// cacheable mapping or writecombine mapping
+	if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) {
+#ifdef DMA_ATTR_NON_CONSISTENT
+		rknpu_obj->dma_attrs |= DMA_ATTR_NON_CONSISTENT;
+#endif
+#ifdef DMA_ATTR_SYS_CACHE_ONLY
+		rknpu_obj->dma_attrs |= DMA_ATTR_SYS_CACHE_ONLY;
+#elif DMA_ATTR_FORCE_COHERENT
+		// force coherent
+		rknpu_obj->dma_attrs |= DMA_ATTR_FORCE_COHERENT;
+#endif
+	} else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) {
+		rknpu_obj->dma_attrs |= DMA_ATTR_WRITE_COMBINE;
+	}
+
+	if (!(rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING))
+		rknpu_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;
+
+#ifdef DMA_ATTR_SKIP_ZEROING
+	if (!(rknpu_obj->flags & RKNPU_MEM_ZEROING))
+		rknpu_obj->dma_attrs |= DMA_ATTR_SKIP_ZEROING;
+#endif
+
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+	if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
+	    rknpu_dev->iommu_en) {
+		return rknpu_gem_get_pages(rknpu_obj);
+	}
+#endif
+
+	if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
+		gfp_mask |= __GFP_ZERO;
+
+	if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) {
+		gfp_mask &= ~__GFP_HIGHMEM;
+		gfp_mask |= __GFP_DMA32;
+	}
+
+	nr_pages = rknpu_obj->size >> PAGE_SHIFT;
+
+	rknpu_obj->pages = rknpu_gem_alloc_page(nr_pages);
+	if (!rknpu_obj->pages) {
+		LOG_ERROR("failed to allocate pages.\n");
+		return -ENOMEM;
+	}
+
+	rknpu_obj->cookie =
+		dma_alloc_attrs(drm->dev, rknpu_obj->size, &rknpu_obj->dma_addr,
+				gfp_mask, rknpu_obj->dma_attrs);
+	if (!rknpu_obj->cookie) {
+		/*
+		 * when RKNPU_MEM_CONTIGUOUS and IOMMU is available
+		 * try to fallback to allocate non-contiguous buffer
+		 */
+		if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
+		    rknpu_dev->iommu_en) {
+			LOG_DEV_WARN(
+				drm->dev,
+				"try to fallback to allocate non-contiguous %lu buffer.\n",
+				rknpu_obj->size);
+			rknpu_obj->dma_attrs &= ~DMA_ATTR_FORCE_CONTIGUOUS;
+			rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS;
+			rknpu_obj->cookie =
+				dma_alloc_attrs(drm->dev, rknpu_obj->size,
+						&rknpu_obj->dma_addr, gfp_mask,
+						rknpu_obj->dma_attrs);
+			if (!rknpu_obj->cookie) {
+				LOG_DEV_ERROR(
+					drm->dev,
+					"failed to allocate non-contiguous %lu buffer.\n",
+					rknpu_obj->size);
+				goto err_free;
+			}
+		} else {
+			LOG_DEV_ERROR(drm->dev,
+				      "failed to allocate %lu buffer.\n",
+				      rknpu_obj->size);
+			goto err_free;
+		}
+	}
+
+	if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING)
+		rknpu_obj->kv_addr = rknpu_obj->cookie;
+
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		ret = -ENOMEM;
+		goto err_free_dma;
+	}
+
+	ret = dma_get_sgtable_attrs(drm->dev, sgt, rknpu_obj->cookie,
+				    rknpu_obj->dma_addr, rknpu_obj->size,
+				    rknpu_obj->dma_attrs);
+	if (ret < 0) {
+		LOG_DEV_ERROR(drm->dev, "failed to get sgtable.\n");
+		goto err_free_sgt;
+	}
+
+	for_each_sg(sgt->sgl, s, sgt->nents, i) {
+		sg_dma_address(s) = sg_phys(s);
+		LOG_DEBUG(
+			"dma alloc sgt[%d], phys_address: %#llx, length: %u\n",
+			i, (__u64)s->dma_address, s->length);
+	}
+
+	if (drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
+					     nr_pages)) {
+		LOG_DEV_ERROR(drm->dev, "invalid sgtable.\n");
+		ret = -EINVAL;
+		goto err_free_sg_table;
+	}
+
+	rknpu_obj->sgt = sgt;
+
+	return ret;
+
+err_free_sg_table:
+	sg_free_table(sgt);
+err_free_sgt:
+	kfree(sgt);
+err_free_dma:
+	dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie,
+		       rknpu_obj->dma_addr, rknpu_obj->dma_attrs);
+err_free:
+	rknpu_gem_free_page(rknpu_obj->pages);
+
+	return ret;
+}
+
+static void rknpu_gem_free_buf(struct rknpu_gem_object *rknpu_obj)
+{
+	struct drm_device *drm = rknpu_obj->base.dev;
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+	struct rknpu_device *rknpu_dev = drm->dev_private;
+#endif
+
+	if (!rknpu_obj->dma_addr) {
+		LOG_DEBUG("dma handle is invalid.\n");
+		return;
+	}
+
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+	if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
+	    rknpu_dev->iommu_en) {
+		rknpu_gem_put_pages(rknpu_obj);
+		return;
+	}
+#endif
+
+	sg_free_table(rknpu_obj->sgt);
+	kfree(rknpu_obj->sgt);
+
+	dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie,
+		       rknpu_obj->dma_addr, rknpu_obj->dma_attrs);
+
+	rknpu_gem_free_page(rknpu_obj->pages);
+
+	rknpu_obj->dma_addr = 0;
+}
+
+static int rknpu_gem_handle_create(struct drm_gem_object *obj,
+				   struct drm_file *file_priv,
+				   unsigned int *handle)
+{
+	int ret = -EINVAL;
+	/*
+	 * allocate a id of idr table where the obj is registered
+	 * and handle has the id what user can see.
+	 */
+	ret = drm_gem_handle_create(file_priv, obj, handle);
+	if (ret)
+		return ret;
+
+	LOG_DEBUG("gem handle = %#x\n", *handle);
+
+	/* drop reference from allocate - handle holds it now. */
+	rknpu_gem_object_put(obj);
+
+	return 0;
+}
+
+static int rknpu_gem_handle_destroy(struct drm_file *file_priv,
+				    unsigned int handle)
+{
+	return drm_gem_handle_delete(file_priv, handle);
+}
+
+static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm,
+					       unsigned long size)
+{
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	struct drm_gem_object *obj = NULL;
+	gfp_t gfp_mask;
+	int ret = -EINVAL;
+
+	rknpu_obj = kzalloc(sizeof(*rknpu_obj), GFP_KERNEL);
+	if (!rknpu_obj)
+		return ERR_PTR(-ENOMEM);
+
+	rknpu_obj->size = size;
+	obj = &rknpu_obj->base;
+
+	ret = drm_gem_object_init(drm, obj, size);
+	if (ret < 0) {
+		LOG_DEV_ERROR(drm->dev, "failed to initialize gem object\n");
+		kfree(rknpu_obj);
+		return ERR_PTR(ret);
+	}
+
+	gfp_mask = mapping_gfp_mask(obj->filp->f_mapping);
+
+	if (rknpu_obj->flags & RKNPU_MEM_ZEROING)
+		gfp_mask |= __GFP_ZERO;
+
+	if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) {
+		gfp_mask &= ~__GFP_HIGHMEM;
+		gfp_mask |= __GFP_DMA32;
+	}
+
+	mapping_set_gfp_mask(obj->filp->f_mapping, gfp_mask);
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret < 0) {
+		drm_gem_object_release(obj);
+		kfree(rknpu_obj);
+		return ERR_PTR(ret);
+	}
+
+	return rknpu_obj;
+}
+
+struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm,
+						 unsigned int flags,
+						 unsigned long size)
+{
+	struct rknpu_device *rknpu_dev = drm->dev_private;
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	int ret = -EINVAL;
+
+	if (flags & ~(RKNPU_MEM_MASK)) {
+		LOG_DEV_ERROR(drm->dev, "invalid buffer flags: %u\n", flags);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (!size) {
+		LOG_DEV_ERROR(drm->dev, "invalid buffer size: %lu\n", size);
+		return ERR_PTR(-EINVAL);
+	}
+
+	size = roundup(size, PAGE_SIZE);
+
+	rknpu_obj = rknpu_gem_init(drm, size);
+	if (IS_ERR(rknpu_obj))
+		return rknpu_obj;
+
+	if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) {
+		/*
+		 * when no IOMMU is available, all allocated buffers are
+		 * contiguous anyway, so drop RKNPU_MEM_NON_CONTIGUOUS flag
+		 */
+		flags &= ~RKNPU_MEM_NON_CONTIGUOUS;
+		LOG_WARN(
+			"non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n");
+	}
+
+	/* set memory type and cache attribute from user side. */
+	rknpu_obj->flags = flags;
+
+	ret = rknpu_gem_alloc_buf(rknpu_obj);
+	if (ret < 0) {
+		drm_gem_object_release(&rknpu_obj->base);
+		kfree(rknpu_obj);
+		return ERR_PTR(ret);
+	}
+
+	LOG_DEBUG(
+		"create dma addr = %#llx, cookie = 0x%p, size = %lu, attrs = %#lx, flags = %#x\n",
+		(__u64)rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
+		rknpu_obj->dma_attrs, rknpu_obj->flags);
+
+	return rknpu_obj;
+}
+
+void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj)
+{
+	struct drm_gem_object *obj = &rknpu_obj->base;
+
+	LOG_DEBUG(
+		"destroy dma addr = %#llx, cookie = 0x%p, size = %lu, attrs = %#lx, flags = %#x, handle count = %d\n",
+		(__u64)rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size,
+		rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count);
+
+	/*
+	 * do not release memory region from exporter.
+	 *
+	 * the region will be released by exporter
+	 * once dmabuf's refcount becomes 0.
+	 */
+	if (obj->import_attach) {
+		drm_prime_gem_destroy(obj, rknpu_obj->sgt);
+		rknpu_gem_free_page(rknpu_obj->pages);
+	} else {
+		rknpu_gem_free_buf(rknpu_obj);
+	}
+
+	/* release file pointer to gem object. */
+	drm_gem_object_release(obj);
+
+	kfree(rknpu_obj);
+}
+
+int rknpu_gem_create_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_priv)
+{
+	struct rknpu_mem_create *args = data;
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	int ret = -EINVAL;
+
+	rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
+	if (!rknpu_obj) {
+		rknpu_obj =
+			rknpu_gem_object_create(dev, args->flags, args->size);
+		if (IS_ERR(rknpu_obj))
+			return PTR_ERR(rknpu_obj);
+
+		ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv,
+					      &args->handle);
+		if (ret) {
+			rknpu_gem_object_destroy(rknpu_obj);
+			return ret;
+		}
+	}
+
+	// rknpu_gem_object_get(&rknpu_obj->base);
+
+	args->size = rknpu_obj->size;
+	args->obj_addr = (__u64)rknpu_obj;
+	args->dma_addr = rknpu_obj->dma_addr;
+
+	return 0;
+}
+
+int rknpu_gem_map_ioctl(struct drm_device *dev, void *data,
+			struct drm_file *file_priv)
+{
+	struct rknpu_mem_map *args = data;
+
+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
+	return rknpu_gem_dumb_map_offset(file_priv, dev, args->handle,
+					 &args->offset);
+#else
+	return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
+				       &args->offset);
+#endif
+}
+
+int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv)
+{
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	struct rknpu_mem_destroy *args = data;
+
+	rknpu_obj = rknpu_gem_object_find(file_priv, args->handle);
+	if (!rknpu_obj)
+		return -EINVAL;
+
+	// rknpu_gem_object_put(&rknpu_obj->base);
+
+	return rknpu_gem_handle_destroy(file_priv, args->handle);
+}
+
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+/*
+ * __vm_map_pages - maps range of kernel pages into user vma
+ * @vma: user vma to map to
+ * @pages: pointer to array of source kernel pages
+ * @num: number of pages in page array
+ * @offset: user's requested vm_pgoff
+ *
+ * This allows drivers to map range of kernel pages into a user vma.
+ *
+ * Return: 0 on success and error code otherwise.
+ */
+static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
+			  unsigned long num, unsigned long offset)
+{
+	unsigned long count = vma_pages(vma);
+	unsigned long uaddr = vma->vm_start;
+	int ret = -EINVAL, i = 0;
+
+	/* Fail if the user requested offset is beyond the end of the object */
+	if (offset >= num)
+		return -ENXIO;
+
+	/* Fail if the user requested size exceeds available object size */
+	if (count > num - offset)
+		return -ENXIO;
+
+	for (i = 0; i < count; i++) {
+		ret = vm_insert_page(vma, uaddr, pages[offset + i]);
+		if (ret < 0)
+			return ret;
+		uaddr += PAGE_SIZE;
+	}
+
+	return 0;
+}
+
+static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj,
+				struct vm_area_struct *vma)
+{
+	struct drm_device *drm = rknpu_obj->base.dev;
+	int ret = -EINVAL;
+
+	vma->vm_flags |= VM_MIXEDMAP;
+
+	ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages,
+			     vma->vm_pgoff);
+	if (ret < 0)
+		LOG_DEV_ERROR(drm->dev, "failed to map pages into vma: %d\n",
+			      ret);
+
+	return ret;
+}
+#endif
+
+static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj,
+				 struct vm_area_struct *vma)
+{
+	struct drm_device *drm = rknpu_obj->base.dev;
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+	struct rknpu_device *rknpu_dev = drm->dev_private;
+#endif
+	unsigned long vm_size = 0;
+	int ret = -EINVAL;
+
+	/*
+	 * clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
+	 * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
+	 * the whole buffer.
+	 */
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_pgoff = 0;
+
+	vm_size = vma->vm_end - vma->vm_start;
+
+	/* check if user-requested size is valid. */
+	if (vm_size > rknpu_obj->size)
+		return -EINVAL;
+
+#if RKNPU_GEM_ALLOC_FROM_PAGES
+	if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) &&
+	    rknpu_dev->iommu_en) {
+		return rknpu_gem_mmap_pages(rknpu_obj, vma);
+	}
+#endif
+
+	ret = dma_mmap_attrs(drm->dev, vma, rknpu_obj->cookie,
+			     rknpu_obj->dma_addr, rknpu_obj->size,
+			     rknpu_obj->dma_attrs);
+	if (ret < 0) {
+		LOG_DEV_ERROR(drm->dev, "failed to mmap, ret = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+void rknpu_gem_free_object(struct drm_gem_object *obj)
+{
+	rknpu_gem_object_destroy(to_rknpu_obj(obj));
+}
+
+int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm,
+			  struct drm_mode_create_dumb *args)
+{
+	struct rknpu_device *rknpu_dev = drm->dev_private;
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	unsigned int flags = 0;
+	int ret = -EINVAL;
+
+	/*
+	 * allocate memory to be used for framebuffer.
+	 * - this callback would be called by user application
+	 *	with DRM_IOCTL_MODE_CREATE_DUMB command.
+	 */
+	args->pitch = args->width * ((args->bpp + 7) / 8);
+	args->size = args->pitch * args->height;
+
+	if (rknpu_dev->iommu_en)
+		flags = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
+	else
+		flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE;
+
+	rknpu_obj = rknpu_gem_object_create(drm, flags, args->size);
+	if (IS_ERR(rknpu_obj)) {
+		LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n");
+		return PTR_ERR(rknpu_obj);
+	}
+
+	ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv,
+				      &args->handle);
+	if (ret) {
+		rknpu_gem_object_destroy(rknpu_obj);
+		return ret;
+	}
+
+	return 0;
+}
+
+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE
+int rknpu_gem_dumb_map_offset(struct drm_file *file_priv,
+			      struct drm_device *drm, uint32_t handle,
+			      uint64_t *offset)
+{
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	struct drm_gem_object *obj = NULL;
+	int ret = -EINVAL;
+
+	rknpu_obj = rknpu_gem_object_find(file_priv, handle);
+	if (!rknpu_obj)
+		return -EINVAL;
+
+	/* Don't allow imported objects to be mapped */
+	obj = &rknpu_obj->base;
+	if (obj->import_attach)
+		return -EINVAL;
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret)
+		return ret;
+
+	*offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+	return 0;
+}
+#endif
+
+#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE
+vm_fault_t rknpu_gem_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
+	struct drm_device *drm = rknpu_obj->base.dev;
+	unsigned long pfn = 0;
+	pgoff_t page_offset = 0;
+
+	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+
+	if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
+		LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
+		return VM_FAULT_SIGBUS;
+	}
+
+	pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
+	return vmf_insert_mixed(vma, vmf->address,
+				__pfn_to_pfn_t(pfn, PFN_DEV));
+}
+#elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
+int rknpu_gem_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
+	struct drm_device *drm = rknpu_obj->base.dev;
+	unsigned long pfn = 0;
+	pgoff_t page_offset = 0;
+	int ret = -EINVAL;
+
+	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+
+	if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
+		LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
+	ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
+
+out:
+	switch (ret) {
+	case 0:
+	case -ERESTARTSYS:
+	case -EINTR:
+		return VM_FAULT_NOPAGE;
+	case -ENOMEM:
+		return VM_FAULT_OOM;
+	default:
+		return VM_FAULT_SIGBUS;
+	}
+}
+#else
+int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
+	struct drm_device *drm = rknpu_obj->base.dev;
+	unsigned long pfn = 0;
+	pgoff_t page_offset = 0;
+	int ret = -EINVAL;
+
+	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
+		      PAGE_SHIFT;
+
+	if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) {
+		LOG_DEV_ERROR(drm->dev, "invalid page offset\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	pfn = page_to_pfn(rknpu_obj->pages[page_offset]);
+	ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address,
+			      __pfn_to_pfn_t(pfn, PFN_DEV));
+
+out:
+	switch (ret) {
+	case 0:
+	case -ERESTARTSYS:
+	case -EINTR:
+		return VM_FAULT_NOPAGE;
+	case -ENOMEM:
+		return VM_FAULT_OOM;
+	default:
+		return VM_FAULT_SIGBUS;
+	}
+}
+#endif
+
+static int rknpu_gem_mmap_obj(struct drm_gem_object *obj,
+			      struct vm_area_struct *vma)
+{
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
+	int ret = -EINVAL;
+
+	LOG_DEBUG("flags = %#x\n", rknpu_obj->flags);
+
+	/* non-cacheable as default. */
+	if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) {
+		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	} else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) {
+		vma->vm_page_prot =
+			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+	} else {
+		vma->vm_page_prot =
+			pgprot_noncached(vm_get_page_prot(vma->vm_flags));
+	}
+
+	ret = rknpu_gem_mmap_buffer(rknpu_obj, vma);
+	if (ret)
+		goto err_close_vm;
+
+	return 0;
+
+err_close_vm:
+	drm_gem_vm_close(vma);
+
+	return ret;
+}
+
+int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj = NULL;
+	int ret = -EINVAL;
+
+	/* set vm_area_struct. */
+	ret = drm_gem_mmap(filp, vma);
+	if (ret < 0) {
+		LOG_ERROR("failed to mmap, ret = %d\n", ret);
+		return ret;
+	}
+
+	obj = vma->vm_private_data;
+
+	if (obj->import_attach)
+		return dma_buf_mmap(obj->dma_buf, vma, 0);
+
+	return rknpu_gem_mmap_obj(obj, vma);
+}
+
+/* low-level interface prime helpers */
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
+struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev,
+					      struct dma_buf *dma_buf)
+{
+	return drm_gem_prime_import_dev(dev, dma_buf, dev->dev);
+}
+#endif
+
+struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
+	int npages = 0;
+
+	npages = rknpu_obj->size >> PAGE_SHIFT;
+
+#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
+	return drm_prime_pages_to_sg(obj->dev, rknpu_obj->pages, npages);
+#else
+	return drm_prime_pages_to_sg(rknpu_obj->pages, npages);
+#endif
+}
+
+struct drm_gem_object *
+rknpu_gem_prime_import_sg_table(struct drm_device *dev,
+				struct dma_buf_attachment *attach,
+				struct sg_table *sgt)
+{
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	int npages = 0;
+	int ret = -EINVAL;
+
+	rknpu_obj = rknpu_gem_init(dev, attach->dmabuf->size);
+	if (IS_ERR(rknpu_obj)) {
+		ret = PTR_ERR(rknpu_obj);
+		return ERR_PTR(ret);
+	}
+
+	rknpu_obj->dma_addr = sg_dma_address(sgt->sgl);
+
+	npages = rknpu_obj->size >> PAGE_SHIFT;
+	rknpu_obj->pages = rknpu_gem_alloc_page(npages);
+	if (!rknpu_obj->pages) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL,
+					       npages);
+	if (ret < 0)
+		goto err_free_large;
+
+	rknpu_obj->sgt = sgt;
+
+	if (sgt->nents == 1) {
+		/* always physically continuous memory if sgt->nents is 1. */
+		rknpu_obj->flags |= RKNPU_MEM_CONTIGUOUS;
+	} else {
+		/*
+		 * this case could be CONTIG or NONCONTIG type but for now
+		 * sets NONCONTIG.
+		 * TODO. we have to find a way that exporter can notify
+		 * the type of its own buffer to importer.
+		 */
+		rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS;
+	}
+
+	return &rknpu_obj->base;
+
+err_free_large:
+	rknpu_gem_free_page(rknpu_obj->pages);
+err:
+	drm_gem_object_release(&rknpu_obj->base);
+	kfree(rknpu_obj);
+	return ERR_PTR(ret);
+}
+
+void *rknpu_gem_prime_vmap(struct drm_gem_object *obj)
+{
+	struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj);
+
+	if (!rknpu_obj->pages)
+		return NULL;
+
+	return vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
+		    PAGE_KERNEL);
+}
+
+void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+	vunmap(vaddr);
+}
+
+int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	int ret = -EINVAL;
+
+	ret = drm_gem_mmap_obj(obj, obj->size, vma);
+	if (ret < 0)
+		return ret;
+
+	return rknpu_gem_mmap_obj(obj, vma);
+}
+
+int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
+{
+	struct rknpu_gem_object *rknpu_obj = NULL;
+	struct rknpu_mem_sync *args = data;
+	struct scatterlist *sg;
+	dma_addr_t sg_dma_addr;
+	unsigned long length, offset = 0;
+	unsigned long sg_offset, sg_left, size = 0;
+	unsigned long len = 0;
+	int i;
+
+	rknpu_obj = (struct rknpu_gem_object *)args->obj_addr;
+	if (!rknpu_obj)
+		return -EINVAL;
+
+	if (!(rknpu_obj->flags & RKNPU_MEM_CACHEABLE))
+		return -EINVAL;
+
+	if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS)) {
+		if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
+			dma_sync_single_range_for_device(
+				dev->dev, rknpu_obj->dma_addr, args->offset,
+				args->size, DMA_TO_DEVICE);
+		}
+		if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
+			dma_sync_single_range_for_cpu(dev->dev,
+						      rknpu_obj->dma_addr,
+						      args->offset, args->size,
+						      DMA_FROM_DEVICE);
+		}
+	} else {
+		struct drm_device *drm = rknpu_obj->base.dev;
+		struct rknpu_device *rknpu_dev = drm->dev_private;
+
+		WARN_ON(!rknpu_dev->fake_dev);
+
+		length = args->size;
+		offset = args->offset;
+
+		for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents,
+			     i) {
+			len += sg->length;
+			if (len <= offset)
+				continue;
+
+			sg_dma_addr = sg_dma_address(sg);
+			sg_left = len - offset;
+			sg_offset = sg->length - sg_left;
+			size = (length < sg_left) ? length : sg_left;
+
+			if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
+				dma_sync_single_range_for_device(
+					rknpu_dev->fake_dev, sg_dma_addr,
+					sg_offset, size, DMA_TO_DEVICE);
+			}
+
+			if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
+				dma_sync_single_range_for_cpu(
+					rknpu_dev->fake_dev, sg_dma_addr,
+					sg_offset, size, DMA_FROM_DEVICE);
+			}
+
+			offset += size;
+			length -= size;
+
+			if (length == 0)
+				break;
+		}
+	}
+
+	return 0;
+}

drivers/rknpu/rknpu_job.c

@@ -0,0 +1,498 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#include <linux/slab.h>
+#include <linux/delay.h>
+
+#include "rknpu_ioctl.h"
+#include "rknpu_drv.h"
+#include "rknpu_reset.h"
+#include "rknpu_gem.h"
+#include "rknpu_fence.h"
+#include "rknpu_job.h"
+
+#define _REG_READ(base, offset) readl(base + (offset))
+#define _REG_WRITE(base, value, offset) writel(value, base + (offset))
+
+#define REG_READ(offset) _REG_READ(rknpu_dev->base, offset)
+#define REG_WRITE(value, offset) _REG_WRITE(rknpu_dev->base, value, offset)
+
+static void rknpu_job_free(struct rknpu_job *job)
+{
+	struct rknpu_gem_object *task_obj = NULL;
+
+	if (job->fence)
+		dma_fence_put(job->fence);
+
+	task_obj = (struct rknpu_gem_object *)job->args->task_obj_addr;
+	if (task_obj)
+		rknpu_gem_object_put(&task_obj->base);
+
+	if (job->args_owner)
+		kfree(job->args);
+
+	kfree(job);
+}
+
+static int rknpu_job_cleanup(struct rknpu_job *job)
+{
+	rknpu_job_free(job);
+
+	return 0;
+}
+
+static void rknpu_job_cleanup_work(struct work_struct *work)
+{
+	struct rknpu_job *job =
+		container_of(work, struct rknpu_job, cleanup_work);
+
+	rknpu_job_cleanup(job);
+}
+
+static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
+						struct rknpu_submit *args)
+{
+	struct rknpu_job *job = NULL;
+	struct rknpu_gem_object *task_obj = NULL;
+
+	job = kzalloc(sizeof(*job), GFP_KERNEL);
+	if (!job)
+		return NULL;
+
+	job->timestamp = ktime_get();
+	job->rknpu_dev = rknpu_dev;
+
+	task_obj = (struct rknpu_gem_object *)args->task_obj_addr;
+	if (task_obj)
+		rknpu_gem_object_get(&task_obj->base);
+
+	if (!(args->flags & RKNPU_JOB_NONBLOCK)) {
+		job->args = args;
+		job->args_owner = false;
+		return job;
+	}
+
+	job->args = kzalloc(sizeof(*args), GFP_KERNEL);
+	if (!job->args) {
+		kfree(job);
+		return NULL;
+	}
+	*job->args = *args;
+	job->args_owner = true;
+
+	INIT_WORK(&job->cleanup_work, rknpu_job_cleanup_work);
+
+	return job;
+}
+
+static inline int rknpu_job_wait(struct rknpu_job *job)
+{
+	struct rknpu_device *rknpu_dev = job->rknpu_dev;
+	struct rknpu_submit *args = job->args;
+	struct rknpu_task *last_task = NULL;
+	int ret = -EINVAL;
+
+	ret = wait_event_interruptible_timeout(rknpu_dev->job_done_wq,
+					       job->flags & RKNPU_JOB_DONE,
+					       msecs_to_jiffies(args->timeout));
+
+	last_task = job->last_task;
+	if (!last_task)
+		return -EINVAL;
+
+	last_task->int_status = job->int_status;
+
+	if (ret <= 0) {
+		args->task_counter = 0;
+		if (args->flags & RKNPU_JOB_PC) {
+			uint32_t task_status =
+				REG_READ(RKNPU_OFFSET_PC_TASK_STATUS);
+			args->task_counter = (task_status & 0xfff);
+		}
+		return ret < 0 ? ret : -ETIMEDOUT;
+	}
+
+	args->task_counter = args->task_number;
+
+	return 0;
+}
+
+static inline int rknpu_job_commit_pc(struct rknpu_job *job)
+{
+	struct rknpu_device *rknpu_dev = job->rknpu_dev;
+	struct rknpu_submit *args = job->args;
+	struct rknpu_gem_object *task_obj =
+		(struct rknpu_gem_object *)args->task_obj_addr;
+	struct rknpu_task *task_base = NULL;
+	struct rknpu_task *first_task = NULL;
+	struct rknpu_task *last_task = NULL;
+	int task_start = args->task_start;
+	int task_end = args->task_start + args->task_number - 1;
+	int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0;
+
+	if (!task_obj)
+		return -EINVAL;
+
+	if ((task_start + 1) * sizeof(*task_base) > task_obj->size ||
+	    (task_end + 1) * sizeof(*task_base) > task_obj->size)
+		return -EINVAL;
+
+	task_base = task_obj->kv_addr;
+
+	first_task = &task_base[task_start];
+	last_task = &task_base[task_end];
+
+	REG_WRITE(first_task->regcmd_data, RKNPU_OFFSET_PC_DATA_ADDR);
+
+	REG_WRITE(first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT - 1,
+		  RKNPU_OFFSET_PC_DATA_AMOUNT);
+
+	REG_WRITE(last_task->int_mask, RKNPU_OFFSET_INT_MASK);
+
+	REG_WRITE(first_task->int_mask, RKNPU_OFFSET_INT_CLEAR);
+
+	REG_WRITE(((0x6 | task_pp_en) << 12) | args->task_number,
+		  RKNPU_OFFSET_PC_TASK_CONTROL);
+
+	REG_WRITE(0x0, RKNPU_OFFSET_PC_DMA_BASE_ADDR);
+
+	job->first_task = first_task;
+	job->last_task = last_task;
+	job->int_mask = last_task->int_mask;
+
+	REG_WRITE(0x1, RKNPU_OFFSET_PC_OP_EN);
+	REG_WRITE(0x0, RKNPU_OFFSET_PC_OP_EN);
+
+	return 0;
+}
+
+static int rknpu_job_commit(struct rknpu_job *job)
+{
+	struct rknpu_device *rknpu_dev = job->rknpu_dev;
+	struct rknpu_submit *args = job->args;
+
+	// switch to slave mode
+	REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR);
+
+	if (!(args->flags & RKNPU_JOB_PC))
+		return -EINVAL;
+
+	return rknpu_job_commit_pc(job);
+}
+
+static void rknpu_job_next(struct rknpu_device *rknpu_dev)
+{
+	struct rknpu_job *job = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
+
+	if (rknpu_dev->job || list_empty(&rknpu_dev->todo_list)) {
+		spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+		return;
+	}
+
+	job = list_first_entry(&rknpu_dev->todo_list, struct rknpu_job, head);
+
+	list_del_init(&job->head);
+
+	rknpu_dev->job = job;
+
+	spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+
+	job->ret = rknpu_job_commit(job);
+}
+
+static void rknpu_job_done(struct rknpu_job *job, int ret)
+{
+	struct rknpu_device *rknpu_dev = job->rknpu_dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
+	rknpu_dev->job = NULL;
+	spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+
+	job->flags |= RKNPU_JOB_DONE;
+	job->ret = ret;
+
+	if (job->fence)
+		dma_fence_signal(job->fence);
+
+	wake_up(&rknpu_dev->job_done_wq);
+	rknpu_job_next(rknpu_dev);
+
+	if (job->flags & RKNPU_JOB_ASYNC)
+		schedule_work(&job->cleanup_work);
+}
+
+static void rknpu_job_schedule(struct rknpu_job *job)
+{
+	struct rknpu_device *rknpu_dev = job->rknpu_dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
+	list_add_tail(&job->head, &rknpu_dev->todo_list);
+	spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+
+	rknpu_job_next(rknpu_dev);
+}
+
+static void rknpu_job_abort(struct rknpu_job *job)
+{
+	struct rknpu_device *rknpu_dev = job->rknpu_dev;
+	unsigned long flags;
+
+	msleep(100);
+	if (job->ret == -ETIMEDOUT)
+		rknpu_soft_reset(rknpu_dev);
+
+	if (job == rknpu_dev->job) {
+		spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
+		rknpu_dev->job = NULL;
+		spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+	}
+
+	rknpu_job_cleanup(job);
+}
+
+static inline uint32_t rknpu_fuzz_status(uint32_t status)
+{
+	uint32_t fuzz_status = 0;
+
+	if ((status & 0x3) != 0)
+		fuzz_status |= 0x3;
+
+	if ((status & 0xc) != 0)
+		fuzz_status |= 0xc;
+
+	if ((status & 0x30) != 0)
+		fuzz_status |= 0x30;
+
+	if ((status & 0xc0) != 0)
+		fuzz_status |= 0xc0;
+
+	if ((status & 0x300) != 0)
+		fuzz_status |= 0x300;
+
+	if ((status & 0xc00) != 0)
+		fuzz_status |= 0xc00;
+
+	return fuzz_status;
+}
+
+irqreturn_t rknpu_irq_handler(int irq, void *data)
+{
+	struct rknpu_device *rknpu_dev = data;
+	struct rknpu_job *job = rknpu_dev->job;
+	uint32_t status = 0;
+
+	if (!job)
+		return IRQ_HANDLED;
+
+	status = REG_READ(RKNPU_OFFSET_INT_STATUS);
+	REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
+
+	job->int_status = status;
+
+	if (rknpu_fuzz_status(status) != job->int_mask) {
+		LOG_DEBUG("irq: status = %#x, mask = %#x\n", status,
+			  job->int_mask);
+		return IRQ_HANDLED;
+	}
+
+	rknpu_job_done(job, 0);
+
+	return IRQ_HANDLED;
+}
+
+static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev)
+{
+	struct rknpu_job *job = NULL;
+	unsigned long flags;
+	ktime_t now = ktime_get();
+
+	job = rknpu_dev->job;
+	if (job &&
+	    ktime_to_ms(ktime_sub(now, job->timestamp)) >= job->args->timeout) {
+		rknpu_soft_reset(rknpu_dev);
+
+		spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
+		rknpu_dev->job = NULL;
+		spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+
+		do {
+			schedule_work(&job->cleanup_work);
+
+			spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
+
+			if (!list_empty(&rknpu_dev->todo_list)) {
+				job = list_first_entry(&rknpu_dev->todo_list,
+						       struct rknpu_job, head);
+				list_del_init(&job->head);
+			} else {
+				job = NULL;
+			}
+
+			spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
+		} while (job);
+	}
+}
+
+int rknpu_submit_ioctl(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
+{
+	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev);
+	struct rknpu_submit *args = data;
+	struct rknpu_job *job = NULL;
+	int ret = -EINVAL;
+
+	if (args->task_number == 0) {
+		LOG_ERROR("invalid rknpu task number!\n");
+		return -EINVAL;
+	}
+
+	job = rknpu_job_alloc(rknpu_dev, args);
+	if (!job) {
+		LOG_ERROR("failed to allocate rknpu job!\n");
+		return -ENOMEM;
+	}
+
+	if (args->flags & RKNPU_JOB_FENCE) {
+		ret = rknpu_fence_alloc(job);
+		if (ret) {
+			rknpu_job_free(job);
+			return ret;
+		}
+		job->args->fence_fd = rknpu_fence_get_fd(job);
+		args->fence_fd = job->args->fence_fd;
+	}
+
+	if (args->flags & RKNPU_JOB_NONBLOCK) {
+		job->flags |= RKNPU_JOB_ASYNC;
+		rknpu_job_timeout_clean(rknpu_dev);
+		rknpu_job_schedule(job);
+		ret = job->ret;
+		if (ret) {
+			rknpu_job_abort(job);
+			return ret;
+		}
+	} else {
+		rknpu_job_schedule(job);
+		if (args->flags & RKNPU_JOB_PC)
+			job->ret = rknpu_job_wait(job);
+
+		args->task_counter = job->args->task_counter;
+		ret = job->ret;
+		if (!ret)
+			rknpu_job_cleanup(job);
+		else
+			rknpu_job_abort(job);
+	}
+
+	return ret;
+}
+
+int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version)
+{
+	if (version != NULL)
+		*version = REG_READ(RKNPU_OFFSET_VERSION);
+
+	return 0;
+}
+
+int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority,
+			  uint32_t *expect, uint32_t *tw)
+{
+	void __iomem *base = rknpu_dev->bw_priority_base;
+
+	if (!base)
+		return -EINVAL;
+
+	spin_lock(&rknpu_dev->lock);
+
+	if (priority != NULL)
+		*priority = _REG_READ(base, 0x0);
+
+	if (expect != NULL)
+		*expect = _REG_READ(base, 0x8);
+
+	if (tw != NULL)
+		*tw = _REG_READ(base, 0xc);
+
+	spin_unlock(&rknpu_dev->lock);
+
+	return 0;
+}
+
+int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority,
+			  uint32_t expect, uint32_t tw)
+{
+	void __iomem *base = rknpu_dev->bw_priority_base;
+
+	if (!base)
+		return -EINVAL;
+
+	spin_lock(&rknpu_dev->lock);
+
+	if (priority != 0)
+		_REG_WRITE(base, priority, 0x0);
+
+	if (expect != 0)
+		_REG_WRITE(base, expect, 0x8);
+
+	if (tw != 0)
+		_REG_WRITE(base, tw, 0xc);
+
+	spin_unlock(&rknpu_dev->lock);
+
+	return 0;
+}
+
+int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev)
+{
+	spin_lock(&rknpu_dev->lock);
+
+	REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
+	REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT);
+
+	spin_unlock(&rknpu_dev->lock);
+
+	return 0;
+}
+
+int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
+			uint32_t *dt_rd, uint32_t *wd_rd)
+{
+	spin_lock(&rknpu_dev->lock);
+
+	if (dt_wr != NULL)
+		*dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT);
+
+	if (dt_rd != NULL)
+		*dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT);
+
+	if (wd_rd != NULL)
+		*wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT);
+
+	spin_unlock(&rknpu_dev->lock);
+
+	return 0;
+}
+
+int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount)
+{
+	uint32_t dt_wr = 0;
+	uint32_t dt_rd = 0;
+	uint32_t wd_rd = 0;
+	int ret = -EINVAL;
+
+	ret = rknpu_get_rw_amount(rknpu_dev, &dt_wr, &dt_rd, &wd_rd);
+
+	if (amount != NULL)
+		*amount = dt_wr + dt_rd + wd_rd;
+
+	return ret;
+}

drivers/rknpu/rknpu_reset.c

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ * Author: Felix Zeng <felix.zeng@rock-chips.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/iommu.h>
+
+#include "rknpu_reset.h"
+
+static inline struct reset_control *rknpu_reset_control_get(struct device *dev,
+							    const char *name)
+{
+	struct reset_control *rst = NULL;
+
+	rst = devm_reset_control_get(dev, name);
+	if (IS_ERR(rst))
+		LOG_DEV_ERROR(dev,
+			      "failed to get rknpu reset control: %s, %ld\n",
+			      name, PTR_ERR(rst));
+
+	return rst;
+}
+
+int rknpu_reset_get(struct rknpu_device *rknpu_dev)
+{
+	struct reset_control *srst_a = NULL;
+	struct reset_control *srst_h = NULL;
+
+	srst_a = rknpu_reset_control_get(rknpu_dev->dev, "srst_a");
+	if (IS_ERR(srst_a))
+		return PTR_ERR(srst_a);
+
+	rknpu_dev->srst_a = srst_a;
+
+	srst_h = devm_reset_control_get(rknpu_dev->dev, "srst_h");
+	if (IS_ERR(srst_h))
+		return PTR_ERR(srst_h);
+
+	rknpu_dev->srst_h = srst_h;
+
+	return 0;
+}
+
+static int rknpu_reset_assert(struct reset_control *rst)
+{
+	int ret = -EINVAL;
+
+	if (!rst)
+		return -EINVAL;
+
+	ret = reset_control_assert(rst);
+	if (ret < 0) {
+		LOG_ERROR("failed to assert rknpu reset: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rknpu_reset_deassert(struct reset_control *rst)
+{
+	int ret = -EINVAL;
+
+	if (!rst)
+		return -EINVAL;
+
+	ret = reset_control_deassert(rst);
+	if (ret < 0) {
+		LOG_ERROR("failed to deassert rknpu reset: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int rknpu_soft_reset(struct rknpu_device *rknpu_dev)
+{
+	struct iommu_domain *domain = NULL;
+	int ret = -EINVAL;
+
+	if (rknpu_dev->bypass_soft_reset) {
+		LOG_WARN("bypass soft reset\n");
+		return 0;
+	}
+
+	LOG_INFO("soft reset\n");
+
+	ret = rknpu_reset_assert(rknpu_dev->srst_a);
+	ret |= rknpu_reset_assert(rknpu_dev->srst_h);
+
+	udelay(10);
+
+	ret |= rknpu_reset_deassert(rknpu_dev->srst_a);
+	ret |= rknpu_reset_deassert(rknpu_dev->srst_h);
+
+	if (ret) {
+		LOG_DEV_ERROR(rknpu_dev->dev,
+			      "failed to soft reset for rknpu: %d\n", ret);
+		return ret;
+	}
+
+	if (rknpu_dev->iommu_en)
+		domain = iommu_get_domain_for_dev(rknpu_dev->dev);
+
+	if (domain) {
+		iommu_detach_device(domain, rknpu_dev->dev);
+		iommu_attach_device(domain, rknpu_dev->dev);
+	}
+
+	return ret;
+}