diff --git a/drivers/Kconfig b/drivers/Kconfig index 8c97ea01aabd..6a7a4715ab05 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -239,4 +239,7 @@ source "drivers/most/Kconfig" source "drivers/rkflash/Kconfig" source "drivers/rk_nand/Kconfig" + +source "drivers/rknpu/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 18cc78f5385c..080b0bc4d126 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -191,3 +191,4 @@ obj-$(CONFIG_COUNTER) += counter/ obj-$(CONFIG_MOST) += most/ obj-$(CONFIG_RK_FLASH) += rkflash/ obj-$(CONFIG_RK_NAND) += rk_nand/ +obj-$(CONFIG_ROCKCHIP_RKNPU) += rknpu/ diff --git a/drivers/rknpu/Kconfig b/drivers/rknpu/Kconfig new file mode 100644 index 000000000000..42cb4718436e --- /dev/null +++ b/drivers/rknpu/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +menu "RKNPU" + depends on ARCH_ROCKCHIP && DRM + +config ROCKCHIP_RKNPU + tristate "ROCKCHIP_RKNPU" + help + rknpu module. + +endmenu diff --git a/drivers/rknpu/Makefile b/drivers/rknpu/Makefile new file mode 100644 index 000000000000..b2b14a9dab62 --- /dev/null +++ b/drivers/rknpu/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ROCKCHIP_RKNPU) += rknpu.o + +ccflags-y += -I$(src)/include +ccflags-y += -Werror + +rknpu-y += rknpu_drv.o +rknpu-y += rknpu_reset.o +rknpu-y += rknpu_job.o +rknpu-y += rknpu_gem.o +rknpu-y += rknpu_fence.o diff --git a/drivers/rknpu/include/rknpu_drv.h b/drivers/rknpu/include/rknpu_drv.h new file mode 100644 index 000000000000..0e17b86d018e --- /dev/null +++ b/drivers/rknpu/include/rknpu_drv.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#ifndef __LINUX_RKNPU_DRV_H_ +#define __LINUX_RKNPU_DRV_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "rknpu_job.h" +#include "rknpu_fence.h" + +#define DRIVER_NAME "rknpu" +#define DRIVER_DESC "RKNPU driver" +#define DRIVER_DATE "20210824" +#define DRIVER_MAJOR 0 +#define DRIVER_MINOR 5 +#define DRIVER_PATCHLEVEL 0 + +#define LOG_TAG "RKNPU" + +#define LOG_INFO(fmt, args...) pr_info(LOG_TAG ": " fmt, ##args) +#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE +#define LOG_WARN(fmt, args...) pr_warn(LOG_TAG ": " fmt, ##args) +#else +#define LOG_WARN(fmt, args...) pr_warning(LOG_TAG ": " fmt, ##args) +#endif +#define LOG_DEBUG(fmt, args...) DRM_DEBUG_DRIVER(LOG_TAG ": " fmt, ##args) +#define LOG_ERROR(fmt, args...) pr_err(LOG_TAG ": " fmt, ##args) + +#define LOG_DEV_INFO(dev, fmt, args...) dev_info(dev, LOG_TAG ": " fmt, ##args) +#define LOG_DEV_WARN(dev, fmt, args...) dev_warn(dev, LOG_TAG ": " fmt, ##args) +#define LOG_DEV_DEBUG(dev, fmt, args...) \ + DRM_DEV_DEBUG_DRIVER(dev, LOG_TAG ": " fmt, ##args) +#define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args) + +struct rknpu_config { + __u32 bw_priority_addr; + __u32 bw_priority_length; + __u64 dma_mask; +}; + +/** + * RKNPU device + * + * @base: IO mapped base address for device + * @dev: Device instance + * @drm_dev: DRM device instance + */ +struct rknpu_device { + void __iomem *base; + struct device *dev; + struct device *fake_dev; + struct drm_device *drm_dev; + atomic_t sequence; + spinlock_t lock; + spinlock_t irq_lock; + struct list_head todo_list; + wait_queue_head_t job_done_wq; + const struct rknpu_config *config; + void __iomem *bw_priority_base; + struct rknpu_job *job; + struct rknpu_fence_context *fence_ctx; + bool iommu_en; + struct reset_control *srst_a; + struct reset_control *srst_h; + struct clk_bulk_data *clks; + int num_clks; + struct regulator *vdd; + struct monitor_dev_info *mdev_info; + struct ipa_power_model_data *model_data; + struct thermal_cooling_device *devfreq_cooling; + struct devfreq *devfreq; + unsigned long current_freq; + unsigned long current_volt; + int bypass_irq_handler; + int bypass_soft_reset; +}; + +#endif /* __LINUX_RKNPU_DRV_H_ */ diff --git a/drivers/rknpu/include/rknpu_fence.h b/drivers/rknpu/include/rknpu_fence.h new file mode 100644 index 000000000000..53047ad0b6a4 --- /dev/null +++ b/drivers/rknpu/include/rknpu_fence.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#ifndef __LINUX_RKNPU_FENCE_H_ +#define __LINUX_RKNPU_FENCE_H_ + +#include "rknpu_job.h" + +struct rknpu_fence_context { + unsigned int context; + unsigned int seqno; + spinlock_t spinlock; +}; + +struct rknpu_fence_context *rknpu_fence_context_alloc(void); + +void rknpu_fence_context_free(struct rknpu_fence_context *fence_ctx); + +int rknpu_fence_alloc(struct rknpu_job *job); + +int rknpu_fence_get_fd(struct rknpu_job *job); + +#endif /* __LINUX_RKNPU_FENCE_H_ */ diff --git a/drivers/rknpu/include/rknpu_gem.h b/drivers/rknpu/include/rknpu_gem.h new file mode 100644 index 000000000000..5d911e98d412 --- /dev/null +++ b/drivers/rknpu/include/rknpu_gem.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#ifndef __LINUX_RKNPU_GEM_H +#define __LINUX_RKNPU_GEM_H + +#include +#include + +#include +#include +#include +#include + +#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE +#include +#endif + +#define to_rknpu_obj(x) container_of(x, struct rknpu_gem_object, base) + +/* + * rknpu drm buffer structure. + * + * @base: a gem object. + * - a new handle to this gem object would be created + * by drm_gem_handle_create(). + * @flags: indicate memory type to allocated buffer and cache attribute. + * @size: size requested from user, in bytes and this size is aligned + * in page unit. + * @cookie: cookie returned by dma_alloc_attrs + * @kv_addr: kernel virtual address to allocated memory region. + * @dma_addr: bus address(accessed by dma) to allocated memory region. + * - this address could be physical address without IOMMU and + * device address with IOMMU. + * @pages: Array of backing pages. + * @sgt: Imported sg_table. + * + * P.S. this object would be transferred to user as kms_bo.handle so + * user can access the buffer through kms_bo.handle. + */ +struct rknpu_gem_object { + struct drm_gem_object base; + unsigned int flags; + unsigned long size; + void *cookie; + void __iomem *kv_addr; + dma_addr_t dma_addr; + unsigned long dma_attrs; + unsigned long num_pages; + struct page **pages; + struct sg_table *sgt; + struct drm_mm_node mm_node; +}; + +/* create a new buffer with gem object */ +struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev, + unsigned int flags, + unsigned long size); + +/* destroy a buffer with gem object */ +void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj); + +/* request gem object creation and buffer allocation as the size */ +int rknpu_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +/* get fake-offset of gem object that can be used with mmap. */ +int rknpu_gem_map_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +/* + * get rknpu drm object, + * gem object reference count would be increased. + */ +static inline void rknpu_gem_object_get(struct drm_gem_object *obj) +{ +#if KERNEL_VERSION(4, 13, 0) < LINUX_VERSION_CODE + drm_gem_object_get(obj); +#else + drm_gem_object_reference(obj); +#endif +} + +/* + * put rknpu drm object acquired from rknpu_gem_object_find() or rknpu_gem_object_get(), + * gem object reference count would be decreased. + */ +static inline void rknpu_gem_object_put(struct drm_gem_object *obj) +{ +#if KERNEL_VERSION(5, 9, 0) <= LINUX_VERSION_CODE + drm_gem_object_put(obj); +#elif KERNEL_VERSION(4, 13, 0) < LINUX_VERSION_CODE + drm_gem_object_put_unlocked(obj); +#else + drm_gem_object_unreference_unlocked(obj); +#endif +} + +/* + * get rknpu drm object from gem handle, this function could be used for + * other drivers such as 2d/3d acceleration drivers. + * with this function call, gem object reference count would be increased. + */ +static inline struct rknpu_gem_object * +rknpu_gem_object_find(struct drm_file *filp, unsigned int handle) +{ + struct drm_gem_object *obj; + + obj = drm_gem_object_lookup(filp, handle); + if (!obj) { + // DRM_ERROR("failed to lookup gem object.\n"); + return NULL; + } + + rknpu_gem_object_put(obj); + + return to_rknpu_obj(obj); +} + +/* get buffer information to memory region allocated by gem. */ +int rknpu_gem_get_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +/* free gem object. */ +void rknpu_gem_free_object(struct drm_gem_object *obj); + +/* create memory region for drm framebuffer. */ +int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, + struct drm_mode_create_dumb *args); + +#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE +/* map memory region for drm framebuffer to user space. */ +int rknpu_gem_dumb_map_offset(struct drm_file *file_priv, + struct drm_device *dev, uint32_t handle, + uint64_t *offset); +#endif + +/* page fault handler and mmap fault address(virtual) to physical memory. */ +#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE +vm_fault_t rknpu_gem_fault(struct vm_fault *vmf); +#elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE +int rknpu_gem_fault(struct vm_fault *vmf); +#else +int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); +#endif + +/* set vm_flags and we can change the vm attribute to other one at here. */ +int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma); + +/* low-level interface prime helpers */ +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE +struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf); +#endif +struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj); +struct drm_gem_object * +rknpu_gem_prime_import_sg_table(struct drm_device *dev, + struct dma_buf_attachment *attach, + struct sg_table *sgt); +void *rknpu_gem_prime_vmap(struct drm_gem_object *obj); +void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr); +int rknpu_gem_prime_mmap(struct drm_gem_object *obj, + struct vm_area_struct *vma); + +int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +static inline void *rknpu_gem_alloc_page(size_t nr_pages) +{ +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE + return kvmalloc_array(nr_pages, sizeof(struct page *), + GFP_KERNEL | __GFP_ZERO); +#else + return drm_calloc_large(nr_pages, sizeof(struct page *)); +#endif +} + +static inline void rknpu_gem_free_page(void *pages) +{ +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE + kvfree(pages); +#else + drm_free_large(pages); +#endif +} + +#endif diff --git a/drivers/rknpu/include/rknpu_ioctl.h b/drivers/rknpu/include/rknpu_ioctl.h new file mode 100644 index 000000000000..a6e59b02924e --- /dev/null +++ b/drivers/rknpu/include/rknpu_ioctl.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#ifndef __LINUX_RKNPU_IOCTL_H +#define __LINUX_RKNPU_IOCTL_H + +#include +#include + +#if !defined(__KERNEL__) +#define __user +#endif + +#define RKNPU_PC_DATA_EXTRA_AMOUNT 4 + +#define RKNPU_OFFSET_VERSION 0x0 +#define RKNPU_OFFSET_PC_OP_EN 0x8 +#define RKNPU_OFFSET_PC_DATA_ADDR 0x10 +#define RKNPU_OFFSET_PC_DATA_AMOUNT 0x14 +#define RKNPU_OFFSET_PC_TASK_CONTROL 0x30 +#define RKNPU_OFFSET_PC_DMA_BASE_ADDR 0x34 +#define RKNPU_OFFSET_PC_TASK_STATUS 0x3c + +#define RKNPU_OFFSET_INT_MASK 0x20 +#define RKNPU_OFFSET_INT_CLEAR 0x24 +#define RKNPU_OFFSET_INT_STATUS 0x28 + +#define RKNPU_OFFSET_CLR_ALL_RW_AMOUNT 0x8010 +#define RKNPU_OFFSET_DT_WR_AMOUNT 0x8034 +#define RKNPU_OFFSET_DT_RD_AMOUNT 0x8038 +#define RKNPU_OFFSET_WT_RD_AMOUNT 0x803c + +#define RKNPU_OFFSET_ENABLE_MASK 0xf008 + +#define RKNPU_INT_CLEAR 0x1ffff + +#define RKNPU_STR_HELPER(x) #x + +#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \ + RKNPU_STR_HELPER(MAJOR) \ + "." RKNPU_STR_HELPER(MINOR) "." RKNPU_STR_HELPER(PATCHLEVEL) +#define RKNPU_GET_DRV_VERSION_CODE(MAJOR, MINOR, PATCHLEVEL) \ + (MAJOR * 10000 + MINOR * 100 + PATCHLEVEL) +#define RKNPU_GET_DRV_VERSION_MAJOR(CODE) (CODE / 10000) +#define RKNPU_GET_DRV_VERSION_MINOR(CODE) ((CODE % 10000) / 100) +#define RKNPU_GET_DRV_VERSION_PATCHLEVEL(CODE) (CODE % 100) + +/* memory type definitions. */ +enum e_rknpu_mem_type { + /* physically continuous memory and used as default. */ + RKNPU_MEM_CONTIGUOUS = 0 << 0, + /* physically non-continuous memory. */ + RKNPU_MEM_NON_CONTIGUOUS = 1 << 0, + /* non-cacheable mapping and used as default. */ + RKNPU_MEM_NON_CACHEABLE = 0 << 1, + /* cacheable mapping. */ + RKNPU_MEM_CACHEABLE = 1 << 1, + /* write-combine mapping. */ + RKNPU_MEM_WRITE_COMBINE = 1 << 2, + /* dma attr kernel mapping */ + RKNPU_MEM_KERNEL_MAPPING = 1 << 3, + /* iommu mapping */ + RKNPU_MEM_IOMMU = 1 << 4, + /* zero mapping */ + RKNPU_MEM_ZEROING = 1 << 5, + /* allocate secure buffer */ + RKNPU_MEM_SECURE = 1 << 6, + /* allocate from non-dma32 zone */ + RKNPU_MEM_NON_DMA32 = 1 << 7, + RKNPU_MEM_MASK = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_CACHEABLE | + RKNPU_MEM_WRITE_COMBINE | RKNPU_MEM_KERNEL_MAPPING | + RKNPU_MEM_IOMMU | RKNPU_MEM_ZEROING | + RKNPU_MEM_SECURE | RKNPU_MEM_NON_DMA32 +}; + +/* sync mode definitions. */ +enum e_rknpu_mem_sync_mode { + RKNPU_MEM_SYNC_TO_DEVICE = 1 << 0, + RKNPU_MEM_SYNC_FROM_DEVICE = 1 << 1, + RKNPU_MEM_SYNC_MASK = + RKNPU_MEM_SYNC_TO_DEVICE | RKNPU_MEM_SYNC_FROM_DEVICE +}; + +/* job mode definitions. */ +enum e_rknpu_job_mode { + RKNPU_JOB_SLAVE = 0 << 0, + RKNPU_JOB_PC = 1 << 0, + RKNPU_JOB_BLOCK = 0 << 1, + RKNPU_JOB_NONBLOCK = 1 << 1, + RKNPU_JOB_PINGPONG = 1 << 2, + RKNPU_JOB_FENCE = 1 << 3, + RKNPU_JOB_MASK = RKNPU_JOB_PC | RKNPU_JOB_NONBLOCK | + RKNPU_JOB_PINGPONG | RKNPU_JOB_FENCE +}; + +/* action definitions */ +enum e_rknpu_action { + RKNPU_GET_HW_VERSION = 0, + RKNPU_GET_DRV_VERSION = 1, + RKNPU_GET_FREQ = 2, + RKNPU_SET_FREQ = 3, + RKNPU_GET_VOLT = 4, + RKNPU_SET_VOLT = 5, + RKNPU_ACT_RESET = 6, + RKNPU_GET_BW_PRIORITY = 7, + RKNPU_SET_BW_PRIORITY = 8, + RKNPU_GET_BW_EXPECT = 9, + RKNPU_SET_BW_EXPECT = 10, + RKNPU_GET_BW_TW = 11, + RKNPU_SET_BW_TW = 12, + RKNPU_ACT_CLR_TOTAL_RW_AMOUNT = 13, + RKNPU_GET_DT_WR_AMOUNT = 14, + RKNPU_GET_DT_RD_AMOUNT = 15, + RKNPU_GET_WT_RD_AMOUNT = 16, + RKNPU_GET_TOTAL_RW_AMOUNT = 17, + RKNPU_GET_IOMMU_EN = 18, + RKNPU_SET_PROC_NICE = 19, +}; + +/** + * User-desired buffer creation information structure. + * + * @handle: The handle of the created GEM object. + * @flags: user request for setting memory type or cache attributes. + * @size: user-desired memory allocation size. + * - this size value would be page-aligned internally. + * @obj_addr: address of RKNPU memory object. + * @dma_addr: dma address that access by rknpu. + */ +struct rknpu_mem_create { + __u32 handle; + __u32 flags; + __u64 size; + __u64 obj_addr; + __u64 dma_addr; +}; + +/** + * A structure for getting a fake-offset that can be used with mmap. + * + * @handle: handle of gem object. + * @reserved: just padding to be 64-bit aligned. + * @offset: a fake-offset of gem object. + */ +struct rknpu_mem_map { + __u32 handle; + __u32 reserved; + __u64 offset; +}; + +/** + * For destroying DMA buffer + * + * @handle: handle of the buffer. + * @reserved: reserved for padding. + */ +struct rknpu_mem_destroy { + __u32 handle; + __u32 reserved; +}; + +/** + * For synchronizing DMA buffer + * + * @flags: user request for setting memory type or cache attributes. + * @reserved: reserved for padding. + * @obj_addr: address of RKNPU memory object. + * @offset: offset in bytes from start address of buffer. + * @size: size of memory region. + * + */ +struct rknpu_mem_sync { + __u32 flags; + __u32 reserved; + __u64 obj_addr; + __u64 offset; + __u64 size; +}; + +/** + * struct rknpu_task structure for task information + * + * @flags: flags for task + * @op_idx: operator index + * @enable_mask: enable mask + * @int_mask: interrupt mask + * @int_clear: interrupt clear + * @int_status: interrupt status + * @regcfg_amount: register config number + * @regcfg_offset: offset for register config + * @regcmd_data: data for register command + * + */ +struct rknpu_task { + __u32 flags; + __u32 op_idx; + __u32 enable_mask; + __u32 int_mask; + __u32 int_clear; + __u32 int_status; + __u32 regcfg_amount; + __u32 regcfg_offset; + __u64 regcmd_data; +} __packed; + +/** + * struct rknpu_submit structure for job submit + * + * @flags: flags for job submit + * @timeout: submit timeout + * @task_start: task start index + * @task_number: task number + * @task_counter: task counter + * @priority: submit priority + * @task_obj_addr: address of task object + * @regcfg_obj_addr: address of register config object + * @user_data: (optional) user data + * @sequence: submit sequence + * @core_id: core id of rknpu + * @fence_fd: dma fence fd + * + */ +struct rknpu_submit { + __u32 flags; + __u32 timeout; + __u32 task_start; + __u32 task_number; + __u32 task_counter; + __s32 priority; + __u64 task_obj_addr; + __u64 regcfg_obj_addr; + __u64 user_data; + __u64 sequence; + __u32 core_id; + __s32 fence_fd; +}; + +/** + * struct rknpu_task structure for action (GET, SET or ACT) + * + * @flags: flags for action + * @value: GET or SET value + * + */ +struct rknpu_action { + __u32 flags; + __u32 value; +}; + +#define RKNPU_ACTION 0x00 +#define RKNPU_SUBMIT 0x01 +#define RKNPU_MEM_CREATE 0x02 +#define RKNPU_MEM_MAP 0x03 +#define RKNPU_MEM_DESTROY 0x04 +#define RKNPU_MEM_SYNC 0x05 + +#if defined(__arm__) || defined(__aarch64__) + +#include + +#define DRM_IOCTL_RKNPU_ACTION \ + DRM_IOWR(DRM_COMMAND_BASE + RKNPU_ACTION, struct rknpu_action) +#define DRM_IOCTL_RKNPU_SUBMIT \ + DRM_IOWR(DRM_COMMAND_BASE + RKNPU_SUBMIT, struct rknpu_submit) +#define DRM_IOCTL_RKNPU_MEM_CREATE \ + DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_CREATE, struct rknpu_mem_create) +#define DRM_IOCTL_RKNPU_MEM_MAP \ + DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_MAP, struct rknpu_mem_map) +#define DRM_IOCTL_RKNPU_MEM_DESTROY \ + DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_DESTROY, struct rknpu_mem_destroy) +#define DRM_IOCTL_RKNPU_MEM_SYNC \ + DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_SYNC, struct rknpu_mem_sync) + +#endif + +#endif diff --git a/drivers/rknpu/include/rknpu_job.h b/drivers/rknpu/include/rknpu_job.h new file mode 100644 index 000000000000..de8846f81bfa --- /dev/null +++ b/drivers/rknpu/include/rknpu_job.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#ifndef __LINUX_RKNPU_JOB_H_ +#define __LINUX_RKNPU_JOB_H_ + +#include +#include + +#include + +#include "rknpu_ioctl.h" + +#define RKNPU_JOB_DONE (1 << 0) +#define RKNPU_JOB_ASYNC (1 << 1) + +struct rknpu_job { + struct rknpu_device *rknpu_dev; + struct list_head head; + struct work_struct cleanup_work; + unsigned int flags; + int ret; + struct rknpu_submit *args; + bool args_owner; + struct rknpu_task *first_task; + struct rknpu_task *last_task; + uint32_t int_mask; + uint32_t int_status; + struct dma_fence *fence; + spinlock_t fence_lock; + ktime_t timestamp; +}; + +irqreturn_t rknpu_irq_handler(int irq, void *data); + +int rknpu_submit_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version); + +int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority, + uint32_t *expect, uint32_t *tw); + +int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority, + uint32_t expect, uint32_t tw); + +int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev); + +int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr, + uint32_t *dt_rd, uint32_t *wd_rd); + +int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount); + +#endif /* __LINUX_RKNPU_JOB_H_ */ diff --git a/drivers/rknpu/include/rknpu_reset.h b/drivers/rknpu/include/rknpu_reset.h new file mode 100644 index 000000000000..05c9622e526c --- /dev/null +++ b/drivers/rknpu/include/rknpu_reset.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include + +#include "rknpu_drv.h" + +int rknpu_reset_get(struct rknpu_device *rknpu_dev); + +int rknpu_soft_reset(struct rknpu_device *rknpu_dev); diff --git a/drivers/rknpu/rknpu_drv.c b/drivers/rknpu/rknpu_drv.c new file mode 100644 index 000000000000..22550ba58dff --- /dev/null +++ b/drivers/rknpu/rknpu_drv.c @@ -0,0 +1,833 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "rknpu_ioctl.h" +#include "rknpu_reset.h" +#include "rknpu_gem.h" +#include "rknpu_fence.h" +#include "rknpu_drv.h" + +static int bypass_irq_handler; +module_param(bypass_irq_handler, int, 0644); +MODULE_PARM_DESC(bypass_irq_handler, + "bypass RKNPU irq handler if set it to 1, disabled by default"); + +static int bypass_soft_reset; +module_param(bypass_soft_reset, int, 0644); +MODULE_PARM_DESC(bypass_soft_reset, + "bypass RKNPU soft reset if set it to 1, disabled by default"); + +static const struct rknpu_config rk356x_rknpu_config = { + .bw_priority_addr = 0xfe180008, + .bw_priority_length = 0x10, + .dma_mask = DMA_BIT_MASK(32), +}; + +/* driver probe and init */ +static const struct of_device_id rknpu_of_match[] = { + { + .compatible = "rockchip,rknpu", + .data = &rk356x_rknpu_config, + }, + { + .compatible = "rockchip,rk3568-rknpu", + .data = &rk356x_rknpu_config, + }, + {}, +}; + +static const struct vm_operations_struct rknpu_gem_vm_ops = { + .fault = rknpu_gem_fault, + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, +}; + +static int rknpu_action_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +static const struct drm_ioctl_desc rknpu_ioctls[] = { + DRM_IOCTL_DEF_DRV(RKNPU_ACTION, rknpu_action_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RKNPU_SUBMIT, rknpu_submit_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RKNPU_MEM_CREATE, rknpu_gem_create_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RKNPU_MEM_MAP, rknpu_gem_map_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RKNPU_MEM_DESTROY, rknpu_gem_destroy_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(RKNPU_MEM_SYNC, rknpu_gem_sync_ioctl, + DRM_RENDER_ALLOW), +}; + +static const struct file_operations rknpu_drm_driver_fops = { + .owner = THIS_MODULE, + .open = drm_open, + .mmap = rknpu_gem_mmap, + .poll = drm_poll, + .read = drm_read, + .unlocked_ioctl = drm_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = drm_compat_ioctl, +#endif + .release = drm_release, + .llseek = noop_llseek, +}; + +static struct drm_driver rknpu_drm_driver = { +#if KERNEL_VERSION(5, 4, 0) <= LINUX_VERSION_CODE + .driver_features = DRIVER_GEM | DRIVER_RENDER, +#else + .driver_features = DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER, +#endif + .gem_free_object_unlocked = rknpu_gem_free_object, + .gem_vm_ops = &rknpu_gem_vm_ops, + .dumb_create = rknpu_gem_dumb_create, +#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE + .dumb_map_offset = rknpu_gem_dumb_map_offset, +#else + .dumb_map_offset = drm_gem_dumb_map_offset, +#endif + .dumb_destroy = drm_gem_dumb_destroy, + .prime_handle_to_fd = drm_gem_prime_handle_to_fd, + .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_export = drm_gem_prime_export, +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE + .gem_prime_import = rknpu_gem_prime_import, +#else + .gem_prime_import = drm_gem_prime_import, +#endif + .gem_prime_get_sg_table = rknpu_gem_prime_get_sg_table, + .gem_prime_import_sg_table = rknpu_gem_prime_import_sg_table, + .gem_prime_vmap = rknpu_gem_prime_vmap, + .gem_prime_vunmap = rknpu_gem_prime_vunmap, + .gem_prime_mmap = rknpu_gem_prime_mmap, + .ioctls = rknpu_ioctls, + .num_ioctls = ARRAY_SIZE(rknpu_ioctls), + .fops = &rknpu_drm_driver_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = DRIVER_DATE, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +static int rknpu_get_drv_version(uint32_t *version) +{ + *version = RKNPU_GET_DRV_VERSION_CODE(DRIVER_MAJOR, DRIVER_MINOR, + DRIVER_PATCHLEVEL); + return 0; +} + +static int rknpu_action_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); + struct rknpu_action *args = data; + int ret = -EINVAL; + + switch (args->flags) { + case RKNPU_GET_HW_VERSION: + ret = rknpu_get_hw_version(rknpu_dev, &args->value); + break; + case RKNPU_GET_DRV_VERSION: + ret = rknpu_get_drv_version(&args->value); + break; + case RKNPU_GET_FREQ: + args->value = rknpu_dev->current_freq; + ret = 0; + break; + case RKNPU_SET_FREQ: + break; + case RKNPU_GET_VOLT: + args->value = rknpu_dev->current_volt; + ret = 0; + break; + case RKNPU_SET_VOLT: + break; + case RKNPU_ACT_RESET: + ret = rknpu_soft_reset(rknpu_dev); + break; + case RKNPU_GET_BW_PRIORITY: + ret = rknpu_get_bw_priority(rknpu_dev, &args->value, NULL, + NULL); + break; + case RKNPU_SET_BW_PRIORITY: + ret = rknpu_set_bw_priority(rknpu_dev, args->value, 0, 0); + break; + case RKNPU_GET_BW_EXPECT: + ret = rknpu_get_bw_priority(rknpu_dev, NULL, &args->value, + NULL); + break; + case RKNPU_SET_BW_EXPECT: + ret = rknpu_set_bw_priority(rknpu_dev, 0, args->value, 0); + break; + case RKNPU_GET_BW_TW: + ret = rknpu_get_bw_priority(rknpu_dev, NULL, NULL, + &args->value); + break; + case RKNPU_SET_BW_TW: + ret = rknpu_set_bw_priority(rknpu_dev, 0, 0, args->value); + break; + case RKNPU_ACT_CLR_TOTAL_RW_AMOUNT: + ret = rknpu_clear_rw_amount(rknpu_dev); + break; + case RKNPU_GET_DT_WR_AMOUNT: + ret = rknpu_get_rw_amount(rknpu_dev, &args->value, NULL, NULL); + break; + case RKNPU_GET_DT_RD_AMOUNT: + ret = rknpu_get_rw_amount(rknpu_dev, NULL, &args->value, NULL); + break; + case RKNPU_GET_WT_RD_AMOUNT: + ret = rknpu_get_rw_amount(rknpu_dev, NULL, NULL, &args->value); + break; + case RKNPU_GET_TOTAL_RW_AMOUNT: + ret = rknpu_get_total_rw_amount(rknpu_dev, &args->value); + break; + case RKNPU_GET_IOMMU_EN: + args->value = rknpu_dev->iommu_en; + ret = 0; + break; + case RKNPU_SET_PROC_NICE: + set_user_nice(current, *(int32_t *)&args->value); + ret = 0; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static bool rknpu_is_iommu_enable(struct device *dev) +{ + struct device_node *iommu = NULL; + + iommu = of_parse_phandle(dev->of_node, "iommus", 0); + if (!iommu) { + LOG_DEV_INFO( + dev, + "rknpu iommu device-tree entry not found!, using non-iommu mode\n"); + return false; + } + + if (!of_device_is_available(iommu)) { + LOG_DEV_INFO(dev, + "rknpu iommu is disabled, using non-iommu mode\n"); + of_node_put(iommu); + return false; + } + of_node_put(iommu); + + LOG_DEV_INFO(dev, "rknpu iommu is enabled, using iommu mode\n"); + + return true; +} + +static int drm_fake_dev_register(struct rknpu_device *rknpu_dev) +{ + const struct platform_device_info rknpu_dev_info = { + .name = "rknpu_dev", + .id = PLATFORM_DEVID_AUTO, + .dma_mask = rknpu_dev->config->dma_mask, + }; + struct platform_device *pdev = NULL; + int ret = -EINVAL; + + pdev = platform_device_register_full(&rknpu_dev_info); + if (pdev) { + ret = of_dma_configure(&pdev->dev, NULL, true); + if (ret) { + platform_device_unregister(pdev); + pdev = NULL; + } + } + + rknpu_dev->fake_dev = pdev ? &pdev->dev : NULL; + + return ret; +} + +static void drm_fake_dev_unregister(struct rknpu_device *rknpu_dev) +{ + struct platform_device *pdev = NULL; + + if (!rknpu_dev->fake_dev) + return; + + pdev = to_platform_device(rknpu_dev->fake_dev); + + platform_device_unregister(pdev); +} + +static int rknpu_drm_probe(struct rknpu_device *rknpu_dev) +{ + struct device *dev = rknpu_dev->dev; + struct drm_device *drm_dev = NULL; + int ret = -EINVAL; + + drm_dev = drm_dev_alloc(&rknpu_drm_driver, dev); + if (IS_ERR(drm_dev)) + return PTR_ERR(drm_dev); + + /* register the DRM device */ + ret = drm_dev_register(drm_dev, 0); + if (ret < 0) + goto err_free_drm; + + drm_dev->dev_private = rknpu_dev; + rknpu_dev->drm_dev = drm_dev; + + drm_fake_dev_register(rknpu_dev); + + return 0; + +err_free_drm: +#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE + drm_dev_put(drm_dev); +#else + drm_dev_unref(drm_dev); +#endif + + return ret; +} + +static void rknpu_drm_remove(struct rknpu_device *rknpu_dev) +{ + struct drm_device *drm_dev = rknpu_dev->drm_dev; + + drm_fake_dev_unregister(rknpu_dev); + + drm_dev_unregister(drm_dev); + +#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE + drm_dev_put(drm_dev); +#else + drm_dev_unref(drm_dev); +#endif +} + +static int rknpu_power_on(struct rknpu_device *rknpu_dev) +{ + struct device *dev = rknpu_dev->dev; + int ret = -EINVAL; + + ret = regulator_enable(rknpu_dev->vdd); + if (ret) { + LOG_DEV_ERROR( + dev, "failed to enable regulator for rknpu, ret = %d\n", + ret); + return ret; + } + + ret = clk_bulk_prepare_enable(rknpu_dev->num_clks, rknpu_dev->clks); + if (ret) { + LOG_DEV_ERROR(dev, "failed to enable clk for rknpu, ret = %d\n", + ret); + return ret; + } + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + LOG_DEV_ERROR(dev, + "failed to get pm runtime for rknpu, ret = %d\n", + ret); + return ret; + } + + return ret; +} + +static int rknpu_power_off(struct rknpu_device *rknpu_dev) +{ + struct device *dev = rknpu_dev->dev; + + pm_runtime_put_sync(dev); + + clk_bulk_disable_unprepare(rknpu_dev->num_clks, rknpu_dev->clks); + + regulator_disable(rknpu_dev->vdd); + + return 0; +} + +static struct monitor_dev_profile npu_mdevp = { + .type = MONITOR_TPYE_DEV, + .low_temp_adjust = rockchip_monitor_dev_low_temp_adjust, + .high_temp_adjust = rockchip_monitor_dev_high_temp_adjust, +}; + +static int npu_devfreq_target(struct device *dev, unsigned long *target_freq, + u32 flags) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + struct dev_pm_opp *opp = NULL; + unsigned long freq = *target_freq; + unsigned long old_freq = rknpu_dev->current_freq; + unsigned long volt, old_volt = rknpu_dev->current_volt; + int ret = -EINVAL; + +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_lock(); +#endif + + opp = devfreq_recommended_opp(dev, &freq, flags); + if (IS_ERR(opp)) { +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_unlock(); +#endif + LOG_DEV_ERROR(dev, "failed to get opp (%ld)\n", PTR_ERR(opp)); + return PTR_ERR(opp); + } + volt = dev_pm_opp_get_voltage(opp); +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_unlock(); +#endif + + /* + * Only update if there is a change of frequency + */ + if (old_freq == freq) { + *target_freq = freq; + if (old_volt == volt) + return 0; + ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt); + return ret; + } + rknpu_dev->current_volt = volt; + return 0; + } + + if (rknpu_dev->vdd && old_volt != volt && old_freq < freq) { + ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, "failed to increase volt %lu\n", + volt); + return ret; + } + } + LOG_DEV_DEBUG(dev, "%luHz %luuV -> %luHz %luuV\n", old_freq, old_volt, + freq, volt); + ret = clk_set_rate(rknpu_dev->clks[0].clk, freq); + if (ret) { + LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq); + return ret; + } + *target_freq = freq; + rknpu_dev->current_freq = freq; + + if (rknpu_dev->devfreq) + rknpu_dev->devfreq->last_status.current_frequency = freq; + + if (rknpu_dev->vdd && old_volt != volt && old_freq > freq) { + ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, "failed to decrease volt %lu\n", + volt); + return ret; + } + } + rknpu_dev->current_volt = volt; + + return ret; +} + +static int npu_devfreq_get_dev_status(struct device *dev, + struct devfreq_dev_status *stat) +{ + return 0; +} + +static int npu_devfreq_get_cur_freq(struct device *dev, unsigned long *freq) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + *freq = rknpu_dev->current_freq; + + return 0; +} + +static struct devfreq_dev_profile npu_devfreq_profile = { + .polling_ms = 50, + .target = npu_devfreq_target, + .get_dev_status = npu_devfreq_get_dev_status, + .get_cur_freq = npu_devfreq_get_cur_freq, +}; + +static unsigned long npu_get_static_power(struct devfreq *devfreq, + unsigned long voltage) +{ + struct device *dev = devfreq->dev.parent; + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev); + + if (!rknpu_dev->model_data) + return 0; + + return rockchip_ipa_get_static_power(rknpu_dev->model_data, voltage); +} + +static struct devfreq_cooling_power npu_cooling_power = { + .get_static_power = &npu_get_static_power, +}; + +static int npu_devfreq_adjust_current_freq_volt(struct device *dev, + struct rknpu_device *rknpu_dev) +{ + unsigned long volt, old_freq, freq; + struct dev_pm_opp *opp = NULL; + int ret = -EINVAL; + + old_freq = clk_get_rate(rknpu_dev->clks[0].clk); + freq = old_freq; + +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_lock(); +#endif + + opp = devfreq_recommended_opp(dev, &freq, 0); + volt = dev_pm_opp_get_voltage(opp); + +#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE + rcu_read_unlock(); +#endif + + if (freq >= old_freq && rknpu_dev->vdd) { + ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt); + return ret; + } + } + LOG_DEV_DEBUG(dev, "adjust current freq=%luHz, volt=%luuV\n", freq, + volt); + ret = clk_set_rate(rknpu_dev->clks[0].clk, freq); + if (ret) { + LOG_DEV_ERROR(dev, "failed to set clock %lu\n", freq); + return ret; + } + if (freq < old_freq && rknpu_dev->vdd) { + ret = regulator_set_voltage(rknpu_dev->vdd, volt, INT_MAX); + if (ret) { + LOG_DEV_ERROR(dev, "failed to set volt %lu\n", volt); + return ret; + } + } + rknpu_dev->current_freq = freq; + rknpu_dev->current_volt = volt; + + return 0; +} + +static int rknpu_devfreq_init(struct rknpu_device *rknpu_dev) +{ + struct device *dev = rknpu_dev->dev; + struct devfreq_dev_profile *dp = &npu_devfreq_profile; + int ret = -EINVAL; + + ret = rockchip_init_opp_table(dev, NULL, "npu_leakage", "rknpu"); + + if (ret) { + LOG_DEV_ERROR(dev, "failed to init_opp_table\n"); + return ret; + } + + ret = npu_devfreq_adjust_current_freq_volt(dev, rknpu_dev); + if (ret) { + LOG_DEV_ERROR(dev, "failed to adjust current freq volt\n"); + return ret; + } + dp->initial_freq = rknpu_dev->current_freq; + + rknpu_dev->devfreq = + devm_devfreq_add_device(dev, dp, "userspace", NULL); + if (IS_ERR(rknpu_dev->devfreq)) { + LOG_DEV_ERROR(dev, "failed to add devfreq\n"); + return PTR_ERR(rknpu_dev->devfreq); + } + devm_devfreq_register_opp_notifier(dev, rknpu_dev->devfreq); + + rknpu_dev->devfreq->last_status.current_frequency = dp->initial_freq; + rknpu_dev->devfreq->last_status.total_time = 1; + rknpu_dev->devfreq->last_status.busy_time = 1; + + npu_mdevp.data = rknpu_dev->devfreq; + rknpu_dev->mdev_info = + rockchip_system_monitor_register(dev, &npu_mdevp); + if (IS_ERR(rknpu_dev->mdev_info)) { + LOG_DEV_DEBUG(dev, "without system monitor\n"); + rknpu_dev->mdev_info = NULL; + } + + of_property_read_u32(dev->of_node, "dynamic-power-coefficient", + (u32 *)&npu_cooling_power.dyn_power_coeff); + +#if KERNEL_VERSION(4, 4, 179) <= LINUX_VERSION_CODE + rknpu_dev->model_data = + rockchip_ipa_power_model_init(dev, "npu_leakage"); + if (IS_ERR_OR_NULL(rknpu_dev->model_data)) { +#else + ret = rockchip_ipa_power_model_init(dev, &rknpu_dev->model_data); + if (ret) { +#endif + rknpu_dev->model_data = NULL; + LOG_DEV_ERROR(dev, "failed to initialize power model\n"); + } else if (rknpu_dev->model_data->dynamic_coefficient) { + npu_cooling_power.dyn_power_coeff = + rknpu_dev->model_data->dynamic_coefficient; + } + + if (!npu_cooling_power.dyn_power_coeff) { + LOG_DEV_ERROR(dev, "failed to get dynamic-coefficient\n"); + goto out; + } + +#if KERNEL_VERSION(4, 4, 179) > LINUX_VERSION_CODE + rockchip_of_get_leakage(dev, "npu_leakage", + &rknpu_dev->model_data->leakage); +#endif + rknpu_dev->devfreq_cooling = of_devfreq_cooling_register_power( + dev->of_node, rknpu_dev->devfreq, &npu_cooling_power); + if (IS_ERR_OR_NULL(rknpu_dev->devfreq_cooling)) + LOG_DEV_ERROR(dev, "failed to register cooling device\n"); + +out: + return 0; +} + +static int rknpu_probe(struct platform_device *pdev) +{ + struct resource *res = NULL; + struct rknpu_device *rknpu_dev = NULL; + struct device *dev = &pdev->dev; + const struct of_device_id *match = NULL; + const struct rknpu_config *config = NULL; + int ret = -EINVAL; + + if (!pdev->dev.of_node) { + LOG_DEV_ERROR(dev, "rknpu device-tree data is missing!\n"); + return -ENODEV; + } + + match = of_match_device(rknpu_of_match, dev); + if (!match) { + LOG_DEV_ERROR(dev, "rknpu device-tree entry is missing!\n"); + return -ENODEV; + } + + rknpu_dev = devm_kzalloc(dev, sizeof(*rknpu_dev), GFP_KERNEL); + if (!rknpu_dev) { + LOG_DEV_ERROR(dev, "failed to allocate rknpu device!\n"); + return -ENOMEM; + } + + config = of_device_get_match_data(dev); + if (!config) + return -EINVAL; + + rknpu_dev->config = config; + rknpu_dev->dev = dev; + + rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev); + if (!rknpu_dev->iommu_en) { + /* Initialize reserved memory resources */ + ret = of_reserved_mem_device_init(dev); + if (!ret) { + LOG_DEV_INFO( + dev, + "initialize reserved memory for rknpu device!\n"); + } + } + + rknpu_dev->bypass_irq_handler = bypass_irq_handler; + rknpu_dev->bypass_soft_reset = bypass_soft_reset; + + rknpu_reset_get(rknpu_dev); + + rknpu_dev->num_clks = devm_clk_bulk_get_all(dev, &rknpu_dev->clks); + if (rknpu_dev->num_clks < 1) { + LOG_DEV_ERROR(dev, "failed to get clk source for rknpu\n"); + return -ENODEV; + } + + rknpu_dev->vdd = devm_regulator_get_optional(dev, "rknpu"); + if (IS_ERR(rknpu_dev->vdd)) { + if (PTR_ERR(rknpu_dev->vdd) != -ENODEV) { + ret = PTR_ERR(rknpu_dev->vdd); + LOG_DEV_ERROR( + dev, + "failed to get vdd regulator for rknpu: %d\n", + ret); + return ret; + } + rknpu_dev->vdd = NULL; + } + + spin_lock_init(&rknpu_dev->lock); + spin_lock_init(&rknpu_dev->irq_lock); + INIT_LIST_HEAD(&rknpu_dev->todo_list); + init_waitqueue_head(&rknpu_dev->job_done_wq); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + LOG_DEV_ERROR(dev, "failed to get memory resource for rknpu\n"); + return -ENXIO; + } + + rknpu_dev->base = devm_ioremap_resource(dev, res); + if (PTR_ERR(rknpu_dev->base) == -EBUSY) { + rknpu_dev->base = + devm_ioremap(dev, res->start, resource_size(res)); + } + + if (IS_ERR(rknpu_dev->base)) { + LOG_DEV_ERROR(dev, "failed to remap register for rknpu\n"); + return PTR_ERR(rknpu_dev->base); + } + + rknpu_dev->bw_priority_base = devm_ioremap( + dev, config->bw_priority_addr, config->bw_priority_length); + if (IS_ERR(rknpu_dev->bw_priority_base)) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to remap bw priority register for rknpu\n"); + rknpu_dev->bw_priority_base = NULL; + } + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res) { + LOG_DEV_ERROR(dev, + "failed to get interrupt resource for rknpu\n"); + return -ENXIO; + } + + if (!rknpu_dev->bypass_irq_handler) { + ret = devm_request_irq(dev, res->start, rknpu_irq_handler, + IRQF_SHARED, dev_name(dev), rknpu_dev); + if (ret) { + LOG_DEV_ERROR(dev, "failed to request irq for rknpu\n"); + return ret; + } + } else { + LOG_DEV_WARN(dev, "bypass irq handler!\n"); + } + + ret = rknpu_drm_probe(rknpu_dev); + if (ret) { + LOG_DEV_ERROR(dev, "failed to probe device for rknpu\n"); + return ret; + } + + rknpu_dev->fence_ctx = rknpu_fence_context_alloc(); + if (IS_ERR(rknpu_dev->fence_ctx)) { + LOG_DEV_ERROR(dev, + "failed to allocate fence context for rknpu\n"); + ret = PTR_ERR(rknpu_dev->fence_ctx); + goto err_remove_drm; + } + + platform_set_drvdata(pdev, rknpu_dev); + + rknpu_devfreq_init(rknpu_dev); + + pm_runtime_enable(dev); + + ret = rknpu_power_on(rknpu_dev); + if (ret) + goto err_free_fence_context; + + return 0; + +err_free_fence_context: + rknpu_fence_context_free(rknpu_dev->fence_ctx); +err_remove_drm: + rknpu_drm_remove(rknpu_dev); + + return ret; +} + +static int rknpu_remove(struct platform_device *pdev) +{ + struct rknpu_device *rknpu_dev = platform_get_drvdata(pdev); + + WARN_ON(rknpu_dev->job); + WARN_ON(!list_empty(&rknpu_dev->todo_list)); + + rknpu_drm_remove(rknpu_dev); + + rknpu_fence_context_free(rknpu_dev->fence_ctx); + + rknpu_power_off(rknpu_dev); + + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static struct platform_driver rknpu_driver = { + .probe = rknpu_probe, + .remove = rknpu_remove, + .driver = { + .owner = THIS_MODULE, + .name = "RKNPU", + .of_match_table = of_match_ptr(rknpu_of_match), + }, +}; + +static int rknpu_init(void) +{ + return platform_driver_register(&rknpu_driver); +} + +static void rknpu_exit(void) +{ + platform_driver_unregister(&rknpu_driver); +} + +late_initcall(rknpu_init); +module_exit(rknpu_exit); + +MODULE_DESCRIPTION("RKNPU driver"); +MODULE_AUTHOR("Felix Zeng "); +MODULE_ALIAS("rockchip-rknpu"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(RKNPU_GET_DRV_VERSION_STRING(DRIVER_MAJOR, DRIVER_MINOR, + DRIVER_PATCHLEVEL)); diff --git a/drivers/rknpu/rknpu_fence.c b/drivers/rknpu/rknpu_fence.c new file mode 100644 index 000000000000..a03c72521969 --- /dev/null +++ b/drivers/rknpu/rknpu_fence.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include +#include +#include +#include + +#include "rknpu_drv.h" +#include "rknpu_job.h" + +#include "rknpu_fence.h" + +static const char *rknpu_fence_get_name(struct dma_fence *fence) +{ + return DRIVER_NAME; +} + +static const struct dma_fence_ops rknpu_fence_ops = { + .get_driver_name = rknpu_fence_get_name, + .get_timeline_name = rknpu_fence_get_name, +}; + +struct rknpu_fence_context *rknpu_fence_context_alloc(void) +{ + struct rknpu_fence_context *fence_ctx = NULL; + + fence_ctx = kzalloc(sizeof(*fence_ctx), GFP_KERNEL); + if (!fence_ctx) + return ERR_PTR(-ENOMEM); + + fence_ctx->context = dma_fence_context_alloc(1); + spin_lock_init(&fence_ctx->spinlock); + + return fence_ctx; +} + +void rknpu_fence_context_free(struct rknpu_fence_context *fence_ctx) +{ + if (!IS_ERR(fence_ctx)) + kfree(fence_ctx); +} + +int rknpu_fence_alloc(struct rknpu_job *job) +{ + struct rknpu_fence_context *fence_ctx = job->rknpu_dev->fence_ctx; + struct dma_fence *fence = NULL; + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return -ENOMEM; + + dma_fence_init(fence, &rknpu_fence_ops, &job->fence_lock, + fence_ctx->context, ++fence_ctx->seqno); + + job->fence = fence; + + return 0; +} + +int rknpu_fence_get_fd(struct rknpu_job *job) +{ + struct sync_file *sync_file = NULL; + int fence_fd = -1; + + if (!job->fence) + return -EINVAL; + + fence_fd = get_unused_fd_flags(O_CLOEXEC); + if (fence_fd < 0) + return fence_fd; + + sync_file = sync_file_create(job->fence); + if (!sync_file) + return -ENOMEM; + + fd_install(fence_fd, sync_file->file); + + return fence_fd; +} diff --git a/drivers/rknpu/rknpu_gem.c b/drivers/rknpu/rknpu_gem.c new file mode 100644 index 000000000000..fb4ba709c022 --- /dev/null +++ b/drivers/rknpu/rknpu_gem.c @@ -0,0 +1,983 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include +#include +#include + +#include +#include +#include +#include + +#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE +#include +#endif + +#include "rknpu_drv.h" +#include "rknpu_ioctl.h" +#include "rknpu_gem.h" + +#define RKNPU_GEM_ALLOC_FROM_PAGES 0 + +#if RKNPU_GEM_ALLOC_FROM_PAGES +static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj) +{ + struct drm_device *drm = rknpu_obj->base.dev; + struct scatterlist *s = NULL; + int ret = -EINVAL, i = 0; + + rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base); + if (IS_ERR(rknpu_obj->pages)) + return PTR_ERR(rknpu_obj->pages); + + rknpu_obj->num_pages = rknpu_obj->base.size >> PAGE_SHIFT; + +#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE + rknpu_obj->sgt = drm_prime_pages_to_sg(drm, rknpu_obj->pages, + rknpu_obj->num_pages); +#else + rknpu_obj->sgt = + drm_prime_pages_to_sg(rknpu_obj->pages, rknpu_obj->num_pages); +#endif + if (IS_ERR(rknpu_obj->sgt)) { + ret = PTR_ERR(rknpu_obj->sgt); + goto put_pages; + } + + for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) { + sg_dma_address(s) = sg_phys(s); + LOG_DEBUG( + "gem pages alloc sgt[%d], phys_address: %#llx, length: %#x\n", + i, (__u64)s->dma_address, s->length); + } + + ret = dma_map_sg_attrs(drm->dev, rknpu_obj->sgt->sgl, + rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL, + rknpu_obj->dma_attrs); + if (ret == 0) { + LOG_DEV_ERROR(drm->dev, "failed to map sg table.\n"); + ret = -EFAULT; + goto free_sgt; + } + + if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) { + rknpu_obj->kv_addr = + vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP, + PAGE_KERNEL); + } + + rknpu_obj->dma_addr = (__u64)sg_dma_address(rknpu_obj->sgt->sgl); + + return 0; + +free_sgt: + sg_free_table(rknpu_obj->sgt); + kfree(rknpu_obj->sgt); +put_pages: + drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false); + + return ret; +} + +static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj) +{ + struct drm_device *drm = rknpu_obj->base.dev; + + if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) + vunmap(rknpu_obj->kv_addr); + + dma_map_sg_attrs(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents, + DMA_BIDIRECTIONAL, rknpu_obj->dma_attrs); + drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true); + sg_free_table(rknpu_obj->sgt); + kfree(rknpu_obj->sgt); +} +#endif + +static int rknpu_gem_alloc_buf(struct rknpu_gem_object *rknpu_obj) +{ + struct drm_device *drm = rknpu_obj->base.dev; + struct rknpu_device *rknpu_dev = drm->dev_private; + unsigned int nr_pages = 0; + struct sg_table *sgt = NULL; + struct scatterlist *s = NULL; + gfp_t gfp_mask = GFP_KERNEL; + int ret = -EINVAL, i = 0; + + if (rknpu_obj->dma_addr) { + LOG_DEBUG("buffer already allocated.\n"); + return 0; + } + + rknpu_obj->dma_attrs = 0; + + /* + * if RKNPU_MEM_CONTIGUOUS, fully physically contiguous memory + * region will be allocated else physically contiguous + * as possible. + */ + if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS)) + rknpu_obj->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS; + + // cacheable mapping or writecombine mapping + if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) { +#ifdef DMA_ATTR_NON_CONSISTENT + rknpu_obj->dma_attrs |= DMA_ATTR_NON_CONSISTENT; +#endif +#ifdef DMA_ATTR_SYS_CACHE_ONLY + rknpu_obj->dma_attrs |= DMA_ATTR_SYS_CACHE_ONLY; +#elif DMA_ATTR_FORCE_COHERENT + // force coherent + rknpu_obj->dma_attrs |= DMA_ATTR_FORCE_COHERENT; +#endif + } else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) { + rknpu_obj->dma_attrs |= DMA_ATTR_WRITE_COMBINE; + } + + if (!(rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING)) + rknpu_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING; + +#ifdef DMA_ATTR_SKIP_ZEROING + if (!(rknpu_obj->flags & RKNPU_MEM_ZEROING)) + rknpu_obj->dma_attrs |= DMA_ATTR_SKIP_ZEROING; +#endif + +#if RKNPU_GEM_ALLOC_FROM_PAGES + if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) && + rknpu_dev->iommu_en) { + return rknpu_gem_get_pages(rknpu_obj); + } +#endif + + if (rknpu_obj->flags & RKNPU_MEM_ZEROING) + gfp_mask |= __GFP_ZERO; + + if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) { + gfp_mask &= ~__GFP_HIGHMEM; + gfp_mask |= __GFP_DMA32; + } + + nr_pages = rknpu_obj->size >> PAGE_SHIFT; + + rknpu_obj->pages = rknpu_gem_alloc_page(nr_pages); + if (!rknpu_obj->pages) { + LOG_ERROR("failed to allocate pages.\n"); + return -ENOMEM; + } + + rknpu_obj->cookie = + dma_alloc_attrs(drm->dev, rknpu_obj->size, &rknpu_obj->dma_addr, + gfp_mask, rknpu_obj->dma_attrs); + if (!rknpu_obj->cookie) { + /* + * when RKNPU_MEM_CONTIGUOUS and IOMMU is available + * try to fallback to allocate non-contiguous buffer + */ + if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) && + rknpu_dev->iommu_en) { + LOG_DEV_WARN( + drm->dev, + "try to fallback to allocate non-contiguous %lu buffer.\n", + rknpu_obj->size); + rknpu_obj->dma_attrs &= ~DMA_ATTR_FORCE_CONTIGUOUS; + rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS; + rknpu_obj->cookie = + dma_alloc_attrs(drm->dev, rknpu_obj->size, + &rknpu_obj->dma_addr, gfp_mask, + rknpu_obj->dma_attrs); + if (!rknpu_obj->cookie) { + LOG_DEV_ERROR( + drm->dev, + "failed to allocate non-contiguous %lu buffer.\n", + rknpu_obj->size); + goto err_free; + } + } else { + LOG_DEV_ERROR(drm->dev, + "failed to allocate %lu buffer.\n", + rknpu_obj->size); + goto err_free; + } + } + + if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) + rknpu_obj->kv_addr = rknpu_obj->cookie; + + sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); + if (!sgt) { + ret = -ENOMEM; + goto err_free_dma; + } + + ret = dma_get_sgtable_attrs(drm->dev, sgt, rknpu_obj->cookie, + rknpu_obj->dma_addr, rknpu_obj->size, + rknpu_obj->dma_attrs); + if (ret < 0) { + LOG_DEV_ERROR(drm->dev, "failed to get sgtable.\n"); + goto err_free_sgt; + } + + for_each_sg(sgt->sgl, s, sgt->nents, i) { + sg_dma_address(s) = sg_phys(s); + LOG_DEBUG( + "dma alloc sgt[%d], phys_address: %#llx, length: %u\n", + i, (__u64)s->dma_address, s->length); + } + + if (drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL, + nr_pages)) { + LOG_DEV_ERROR(drm->dev, "invalid sgtable.\n"); + ret = -EINVAL; + goto err_free_sg_table; + } + + rknpu_obj->sgt = sgt; + + return ret; + +err_free_sg_table: + sg_free_table(sgt); +err_free_sgt: + kfree(sgt); +err_free_dma: + dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie, + rknpu_obj->dma_addr, rknpu_obj->dma_attrs); +err_free: + rknpu_gem_free_page(rknpu_obj->pages); + + return ret; +} + +static void rknpu_gem_free_buf(struct rknpu_gem_object *rknpu_obj) +{ + struct drm_device *drm = rknpu_obj->base.dev; +#if RKNPU_GEM_ALLOC_FROM_PAGES + struct rknpu_device *rknpu_dev = drm->dev_private; +#endif + + if (!rknpu_obj->dma_addr) { + LOG_DEBUG("dma handle is invalid.\n"); + return; + } + +#if RKNPU_GEM_ALLOC_FROM_PAGES + if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) && + rknpu_dev->iommu_en) { + rknpu_gem_put_pages(rknpu_obj); + return; + } +#endif + + sg_free_table(rknpu_obj->sgt); + kfree(rknpu_obj->sgt); + + dma_free_attrs(drm->dev, rknpu_obj->size, rknpu_obj->cookie, + rknpu_obj->dma_addr, rknpu_obj->dma_attrs); + + rknpu_gem_free_page(rknpu_obj->pages); + + rknpu_obj->dma_addr = 0; +} + +static int rknpu_gem_handle_create(struct drm_gem_object *obj, + struct drm_file *file_priv, + unsigned int *handle) +{ + int ret = -EINVAL; + /* + * allocate a id of idr table where the obj is registered + * and handle has the id what user can see. + */ + ret = drm_gem_handle_create(file_priv, obj, handle); + if (ret) + return ret; + + LOG_DEBUG("gem handle = %#x\n", *handle); + + /* drop reference from allocate - handle holds it now. */ + rknpu_gem_object_put(obj); + + return 0; +} + +static int rknpu_gem_handle_destroy(struct drm_file *file_priv, + unsigned int handle) +{ + return drm_gem_handle_delete(file_priv, handle); +} + +static struct rknpu_gem_object *rknpu_gem_init(struct drm_device *drm, + unsigned long size) +{ + struct rknpu_gem_object *rknpu_obj = NULL; + struct drm_gem_object *obj = NULL; + gfp_t gfp_mask; + int ret = -EINVAL; + + rknpu_obj = kzalloc(sizeof(*rknpu_obj), GFP_KERNEL); + if (!rknpu_obj) + return ERR_PTR(-ENOMEM); + + rknpu_obj->size = size; + obj = &rknpu_obj->base; + + ret = drm_gem_object_init(drm, obj, size); + if (ret < 0) { + LOG_DEV_ERROR(drm->dev, "failed to initialize gem object\n"); + kfree(rknpu_obj); + return ERR_PTR(ret); + } + + gfp_mask = mapping_gfp_mask(obj->filp->f_mapping); + + if (rknpu_obj->flags & RKNPU_MEM_ZEROING) + gfp_mask |= __GFP_ZERO; + + if (!(rknpu_obj->flags & RKNPU_MEM_NON_DMA32)) { + gfp_mask &= ~__GFP_HIGHMEM; + gfp_mask |= __GFP_DMA32; + } + + mapping_set_gfp_mask(obj->filp->f_mapping, gfp_mask); + + ret = drm_gem_create_mmap_offset(obj); + if (ret < 0) { + drm_gem_object_release(obj); + kfree(rknpu_obj); + return ERR_PTR(ret); + } + + return rknpu_obj; +} + +struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, + unsigned int flags, + unsigned long size) +{ + struct rknpu_device *rknpu_dev = drm->dev_private; + struct rknpu_gem_object *rknpu_obj = NULL; + int ret = -EINVAL; + + if (flags & ~(RKNPU_MEM_MASK)) { + LOG_DEV_ERROR(drm->dev, "invalid buffer flags: %u\n", flags); + return ERR_PTR(-EINVAL); + } + + if (!size) { + LOG_DEV_ERROR(drm->dev, "invalid buffer size: %lu\n", size); + return ERR_PTR(-EINVAL); + } + + size = roundup(size, PAGE_SIZE); + + rknpu_obj = rknpu_gem_init(drm, size); + if (IS_ERR(rknpu_obj)) + return rknpu_obj; + + if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) { + /* + * when no IOMMU is available, all allocated buffers are + * contiguous anyway, so drop RKNPU_MEM_NON_CONTIGUOUS flag + */ + flags &= ~RKNPU_MEM_NON_CONTIGUOUS; + LOG_WARN( + "non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n"); + } + + /* set memory type and cache attribute from user side. */ + rknpu_obj->flags = flags; + + ret = rknpu_gem_alloc_buf(rknpu_obj); + if (ret < 0) { + drm_gem_object_release(&rknpu_obj->base); + kfree(rknpu_obj); + return ERR_PTR(ret); + } + + LOG_DEBUG( + "create dma addr = %#llx, cookie = 0x%p, size = %lu, attrs = %#lx, flags = %#x\n", + (__u64)rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size, + rknpu_obj->dma_attrs, rknpu_obj->flags); + + return rknpu_obj; +} + +void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj) +{ + struct drm_gem_object *obj = &rknpu_obj->base; + + LOG_DEBUG( + "destroy dma addr = %#llx, cookie = 0x%p, size = %lu, attrs = %#lx, flags = %#x, handle count = %d\n", + (__u64)rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size, + rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count); + + /* + * do not release memory region from exporter. + * + * the region will be released by exporter + * once dmabuf's refcount becomes 0. + */ + if (obj->import_attach) { + drm_prime_gem_destroy(obj, rknpu_obj->sgt); + rknpu_gem_free_page(rknpu_obj->pages); + } else { + rknpu_gem_free_buf(rknpu_obj); + } + + /* release file pointer to gem object. */ + drm_gem_object_release(obj); + + kfree(rknpu_obj); +} + +int rknpu_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct rknpu_mem_create *args = data; + struct rknpu_gem_object *rknpu_obj = NULL; + int ret = -EINVAL; + + rknpu_obj = rknpu_gem_object_find(file_priv, args->handle); + if (!rknpu_obj) { + rknpu_obj = + rknpu_gem_object_create(dev, args->flags, args->size); + if (IS_ERR(rknpu_obj)) + return PTR_ERR(rknpu_obj); + + ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv, + &args->handle); + if (ret) { + rknpu_gem_object_destroy(rknpu_obj); + return ret; + } + } + + // rknpu_gem_object_get(&rknpu_obj->base); + + args->size = rknpu_obj->size; + args->obj_addr = (__u64)rknpu_obj; + args->dma_addr = rknpu_obj->dma_addr; + + return 0; +} + +int rknpu_gem_map_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct rknpu_mem_map *args = data; + +#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE + return rknpu_gem_dumb_map_offset(file_priv, dev, args->handle, + &args->offset); +#else + return drm_gem_dumb_map_offset(file_priv, dev, args->handle, + &args->offset); +#endif +} + +int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct rknpu_gem_object *rknpu_obj = NULL; + struct rknpu_mem_destroy *args = data; + + rknpu_obj = rknpu_gem_object_find(file_priv, args->handle); + if (!rknpu_obj) + return -EINVAL; + + // rknpu_gem_object_put(&rknpu_obj->base); + + return rknpu_gem_handle_destroy(file_priv, args->handle); +} + +#if RKNPU_GEM_ALLOC_FROM_PAGES +/* + * __vm_map_pages - maps range of kernel pages into user vma + * @vma: user vma to map to + * @pages: pointer to array of source kernel pages + * @num: number of pages in page array + * @offset: user's requested vm_pgoff + * + * This allows drivers to map range of kernel pages into a user vma. + * + * Return: 0 on success and error code otherwise. + */ +static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, + unsigned long num, unsigned long offset) +{ + unsigned long count = vma_pages(vma); + unsigned long uaddr = vma->vm_start; + int ret = -EINVAL, i = 0; + + /* Fail if the user requested offset is beyond the end of the object */ + if (offset >= num) + return -ENXIO; + + /* Fail if the user requested size exceeds available object size */ + if (count > num - offset) + return -ENXIO; + + for (i = 0; i < count; i++) { + ret = vm_insert_page(vma, uaddr, pages[offset + i]); + if (ret < 0) + return ret; + uaddr += PAGE_SIZE; + } + + return 0; +} + +static int rknpu_gem_mmap_pages(struct rknpu_gem_object *rknpu_obj, + struct vm_area_struct *vma) +{ + struct drm_device *drm = rknpu_obj->base.dev; + int ret = -EINVAL; + + vma->vm_flags |= VM_MIXEDMAP; + + ret = __vm_map_pages(vma, rknpu_obj->pages, rknpu_obj->num_pages, + vma->vm_pgoff); + if (ret < 0) + LOG_DEV_ERROR(drm->dev, "failed to map pages into vma: %d\n", + ret); + + return ret; +} +#endif + +static int rknpu_gem_mmap_buffer(struct rknpu_gem_object *rknpu_obj, + struct vm_area_struct *vma) +{ + struct drm_device *drm = rknpu_obj->base.dev; +#if RKNPU_GEM_ALLOC_FROM_PAGES + struct rknpu_device *rknpu_dev = drm->dev_private; +#endif + unsigned long vm_size = 0; + int ret = -EINVAL; + + /* + * clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the + * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map + * the whole buffer. + */ + vma->vm_flags &= ~VM_PFNMAP; + vma->vm_pgoff = 0; + + vm_size = vma->vm_end - vma->vm_start; + + /* check if user-requested size is valid. */ + if (vm_size > rknpu_obj->size) + return -EINVAL; + +#if RKNPU_GEM_ALLOC_FROM_PAGES + if ((rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS) && + rknpu_dev->iommu_en) { + return rknpu_gem_mmap_pages(rknpu_obj, vma); + } +#endif + + ret = dma_mmap_attrs(drm->dev, vma, rknpu_obj->cookie, + rknpu_obj->dma_addr, rknpu_obj->size, + rknpu_obj->dma_attrs); + if (ret < 0) { + LOG_DEV_ERROR(drm->dev, "failed to mmap, ret = %d\n", ret); + return ret; + } + + return 0; +} + +void rknpu_gem_free_object(struct drm_gem_object *obj) +{ + rknpu_gem_object_destroy(to_rknpu_obj(obj)); +} + +int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, + struct drm_mode_create_dumb *args) +{ + struct rknpu_device *rknpu_dev = drm->dev_private; + struct rknpu_gem_object *rknpu_obj = NULL; + unsigned int flags = 0; + int ret = -EINVAL; + + /* + * allocate memory to be used for framebuffer. + * - this callback would be called by user application + * with DRM_IOCTL_MODE_CREATE_DUMB command. + */ + args->pitch = args->width * ((args->bpp + 7) / 8); + args->size = args->pitch * args->height; + + if (rknpu_dev->iommu_en) + flags = RKNPU_MEM_NON_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE; + else + flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE; + + rknpu_obj = rknpu_gem_object_create(drm, flags, args->size); + if (IS_ERR(rknpu_obj)) { + LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n"); + return PTR_ERR(rknpu_obj); + } + + ret = rknpu_gem_handle_create(&rknpu_obj->base, file_priv, + &args->handle); + if (ret) { + rknpu_gem_object_destroy(rknpu_obj); + return ret; + } + + return 0; +} + +#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE +int rknpu_gem_dumb_map_offset(struct drm_file *file_priv, + struct drm_device *drm, uint32_t handle, + uint64_t *offset) +{ + struct rknpu_gem_object *rknpu_obj = NULL; + struct drm_gem_object *obj = NULL; + int ret = -EINVAL; + + rknpu_obj = rknpu_gem_object_find(file_priv, handle); + if (!rknpu_obj) + return -EINVAL; + + /* Don't allow imported objects to be mapped */ + obj = &rknpu_obj->base; + if (obj->import_attach) + return -EINVAL; + + ret = drm_gem_create_mmap_offset(obj); + if (ret) + return ret; + + *offset = drm_vma_node_offset_addr(&obj->vma_node); + + return 0; +} +#endif + +#if KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE +vm_fault_t rknpu_gem_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *obj = vma->vm_private_data; + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + struct drm_device *drm = rknpu_obj->base.dev; + unsigned long pfn = 0; + pgoff_t page_offset = 0; + + page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + + if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) { + LOG_DEV_ERROR(drm->dev, "invalid page offset\n"); + return VM_FAULT_SIGBUS; + } + + pfn = page_to_pfn(rknpu_obj->pages[page_offset]); + return vmf_insert_mixed(vma, vmf->address, + __pfn_to_pfn_t(pfn, PFN_DEV)); +} +#elif KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE +int rknpu_gem_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct drm_gem_object *obj = vma->vm_private_data; + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + struct drm_device *drm = rknpu_obj->base.dev; + unsigned long pfn = 0; + pgoff_t page_offset = 0; + int ret = -EINVAL; + + page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + + if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) { + LOG_DEV_ERROR(drm->dev, "invalid page offset\n"); + ret = -EINVAL; + goto out; + } + + pfn = page_to_pfn(rknpu_obj->pages[page_offset]); + ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV)); + +out: + switch (ret) { + case 0: + case -ERESTARTSYS: + case -EINTR: + return VM_FAULT_NOPAGE; + case -ENOMEM: + return VM_FAULT_OOM; + default: + return VM_FAULT_SIGBUS; + } +} +#else +int rknpu_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct drm_gem_object *obj = vma->vm_private_data; + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + struct drm_device *drm = rknpu_obj->base.dev; + unsigned long pfn = 0; + pgoff_t page_offset = 0; + int ret = -EINVAL; + + page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >> + PAGE_SHIFT; + + if (page_offset >= (rknpu_obj->size >> PAGE_SHIFT)) { + LOG_DEV_ERROR(drm->dev, "invalid page offset\n"); + ret = -EINVAL; + goto out; + } + + pfn = page_to_pfn(rknpu_obj->pages[page_offset]); + ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, + __pfn_to_pfn_t(pfn, PFN_DEV)); + +out: + switch (ret) { + case 0: + case -ERESTARTSYS: + case -EINTR: + return VM_FAULT_NOPAGE; + case -ENOMEM: + return VM_FAULT_OOM; + default: + return VM_FAULT_SIGBUS; + } +} +#endif + +static int rknpu_gem_mmap_obj(struct drm_gem_object *obj, + struct vm_area_struct *vma) +{ + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + int ret = -EINVAL; + + LOG_DEBUG("flags = %#x\n", rknpu_obj->flags); + + /* non-cacheable as default. */ + if (rknpu_obj->flags & RKNPU_MEM_CACHEABLE) { + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + } else if (rknpu_obj->flags & RKNPU_MEM_WRITE_COMBINE) { + vma->vm_page_prot = + pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); + } else { + vma->vm_page_prot = + pgprot_noncached(vm_get_page_prot(vma->vm_flags)); + } + + ret = rknpu_gem_mmap_buffer(rknpu_obj, vma); + if (ret) + goto err_close_vm; + + return 0; + +err_close_vm: + drm_gem_vm_close(vma); + + return ret; +} + +int rknpu_gem_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct drm_gem_object *obj = NULL; + int ret = -EINVAL; + + /* set vm_area_struct. */ + ret = drm_gem_mmap(filp, vma); + if (ret < 0) { + LOG_ERROR("failed to mmap, ret = %d\n", ret); + return ret; + } + + obj = vma->vm_private_data; + + if (obj->import_attach) + return dma_buf_mmap(obj->dma_buf, vma, 0); + + return rknpu_gem_mmap_obj(obj, vma); +} + +/* low-level interface prime helpers */ +#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE +struct drm_gem_object *rknpu_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf) +{ + return drm_gem_prime_import_dev(dev, dma_buf, dev->dev); +} +#endif + +struct sg_table *rknpu_gem_prime_get_sg_table(struct drm_gem_object *obj) +{ + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + int npages = 0; + + npages = rknpu_obj->size >> PAGE_SHIFT; + +#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE + return drm_prime_pages_to_sg(obj->dev, rknpu_obj->pages, npages); +#else + return drm_prime_pages_to_sg(rknpu_obj->pages, npages); +#endif +} + +struct drm_gem_object * +rknpu_gem_prime_import_sg_table(struct drm_device *dev, + struct dma_buf_attachment *attach, + struct sg_table *sgt) +{ + struct rknpu_gem_object *rknpu_obj = NULL; + int npages = 0; + int ret = -EINVAL; + + rknpu_obj = rknpu_gem_init(dev, attach->dmabuf->size); + if (IS_ERR(rknpu_obj)) { + ret = PTR_ERR(rknpu_obj); + return ERR_PTR(ret); + } + + rknpu_obj->dma_addr = sg_dma_address(sgt->sgl); + + npages = rknpu_obj->size >> PAGE_SHIFT; + rknpu_obj->pages = rknpu_gem_alloc_page(npages); + if (!rknpu_obj->pages) { + ret = -ENOMEM; + goto err; + } + + ret = drm_prime_sg_to_page_addr_arrays(sgt, rknpu_obj->pages, NULL, + npages); + if (ret < 0) + goto err_free_large; + + rknpu_obj->sgt = sgt; + + if (sgt->nents == 1) { + /* always physically continuous memory if sgt->nents is 1. */ + rknpu_obj->flags |= RKNPU_MEM_CONTIGUOUS; + } else { + /* + * this case could be CONTIG or NONCONTIG type but for now + * sets NONCONTIG. + * TODO. we have to find a way that exporter can notify + * the type of its own buffer to importer. + */ + rknpu_obj->flags |= RKNPU_MEM_NON_CONTIGUOUS; + } + + return &rknpu_obj->base; + +err_free_large: + rknpu_gem_free_page(rknpu_obj->pages); +err: + drm_gem_object_release(&rknpu_obj->base); + kfree(rknpu_obj); + return ERR_PTR(ret); +} + +void *rknpu_gem_prime_vmap(struct drm_gem_object *obj) +{ + struct rknpu_gem_object *rknpu_obj = to_rknpu_obj(obj); + + if (!rknpu_obj->pages) + return NULL; + + return vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP, + PAGE_KERNEL); +} + +void rknpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) +{ + vunmap(vaddr); +} + +int rknpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + int ret = -EINVAL; + + ret = drm_gem_mmap_obj(obj, obj->size, vma); + if (ret < 0) + return ret; + + return rknpu_gem_mmap_obj(obj, vma); +} + +int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct rknpu_gem_object *rknpu_obj = NULL; + struct rknpu_mem_sync *args = data; + struct scatterlist *sg; + dma_addr_t sg_dma_addr; + unsigned long length, offset = 0; + unsigned long sg_offset, sg_left, size = 0; + unsigned long len = 0; + int i; + + rknpu_obj = (struct rknpu_gem_object *)args->obj_addr; + if (!rknpu_obj) + return -EINVAL; + + if (!(rknpu_obj->flags & RKNPU_MEM_CACHEABLE)) + return -EINVAL; + + if (!(rknpu_obj->flags & RKNPU_MEM_NON_CONTIGUOUS)) { + if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) { + dma_sync_single_range_for_device( + dev->dev, rknpu_obj->dma_addr, args->offset, + args->size, DMA_TO_DEVICE); + } + if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) { + dma_sync_single_range_for_cpu(dev->dev, + rknpu_obj->dma_addr, + args->offset, args->size, + DMA_FROM_DEVICE); + } + } else { + struct drm_device *drm = rknpu_obj->base.dev; + struct rknpu_device *rknpu_dev = drm->dev_private; + + WARN_ON(!rknpu_dev->fake_dev); + + length = args->size; + offset = args->offset; + + for_each_sg(rknpu_obj->sgt->sgl, sg, rknpu_obj->sgt->nents, + i) { + len += sg->length; + if (len <= offset) + continue; + + sg_dma_addr = sg_dma_address(sg); + sg_left = len - offset; + sg_offset = sg->length - sg_left; + size = (length < sg_left) ? length : sg_left; + + if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) { + dma_sync_single_range_for_device( + rknpu_dev->fake_dev, sg_dma_addr, + sg_offset, size, DMA_TO_DEVICE); + } + + if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) { + dma_sync_single_range_for_cpu( + rknpu_dev->fake_dev, sg_dma_addr, + sg_offset, size, DMA_FROM_DEVICE); + } + + offset += size; + length -= size; + + if (length == 0) + break; + } + } + + return 0; +} diff --git a/drivers/rknpu/rknpu_job.c b/drivers/rknpu/rknpu_job.c new file mode 100644 index 000000000000..6080035519cd --- /dev/null +++ b/drivers/rknpu/rknpu_job.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include +#include + +#include "rknpu_ioctl.h" +#include "rknpu_drv.h" +#include "rknpu_reset.h" +#include "rknpu_gem.h" +#include "rknpu_fence.h" +#include "rknpu_job.h" + +#define _REG_READ(base, offset) readl(base + (offset)) +#define _REG_WRITE(base, value, offset) writel(value, base + (offset)) + +#define REG_READ(offset) _REG_READ(rknpu_dev->base, offset) +#define REG_WRITE(value, offset) _REG_WRITE(rknpu_dev->base, value, offset) + +static void rknpu_job_free(struct rknpu_job *job) +{ + struct rknpu_gem_object *task_obj = NULL; + + if (job->fence) + dma_fence_put(job->fence); + + task_obj = (struct rknpu_gem_object *)job->args->task_obj_addr; + if (task_obj) + rknpu_gem_object_put(&task_obj->base); + + if (job->args_owner) + kfree(job->args); + + kfree(job); +} + +static int rknpu_job_cleanup(struct rknpu_job *job) +{ + rknpu_job_free(job); + + return 0; +} + +static void rknpu_job_cleanup_work(struct work_struct *work) +{ + struct rknpu_job *job = + container_of(work, struct rknpu_job, cleanup_work); + + rknpu_job_cleanup(job); +} + +static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev, + struct rknpu_submit *args) +{ + struct rknpu_job *job = NULL; + struct rknpu_gem_object *task_obj = NULL; + + job = kzalloc(sizeof(*job), GFP_KERNEL); + if (!job) + return NULL; + + job->timestamp = ktime_get(); + job->rknpu_dev = rknpu_dev; + + task_obj = (struct rknpu_gem_object *)args->task_obj_addr; + if (task_obj) + rknpu_gem_object_get(&task_obj->base); + + if (!(args->flags & RKNPU_JOB_NONBLOCK)) { + job->args = args; + job->args_owner = false; + return job; + } + + job->args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!job->args) { + kfree(job); + return NULL; + } + *job->args = *args; + job->args_owner = true; + + INIT_WORK(&job->cleanup_work, rknpu_job_cleanup_work); + + return job; +} + +static inline int rknpu_job_wait(struct rknpu_job *job) +{ + struct rknpu_device *rknpu_dev = job->rknpu_dev; + struct rknpu_submit *args = job->args; + struct rknpu_task *last_task = NULL; + int ret = -EINVAL; + + ret = wait_event_interruptible_timeout(rknpu_dev->job_done_wq, + job->flags & RKNPU_JOB_DONE, + msecs_to_jiffies(args->timeout)); + + last_task = job->last_task; + if (!last_task) + return -EINVAL; + + last_task->int_status = job->int_status; + + if (ret <= 0) { + args->task_counter = 0; + if (args->flags & RKNPU_JOB_PC) { + uint32_t task_status = + REG_READ(RKNPU_OFFSET_PC_TASK_STATUS); + args->task_counter = (task_status & 0xfff); + } + return ret < 0 ? ret : -ETIMEDOUT; + } + + args->task_counter = args->task_number; + + return 0; +} + +static inline int rknpu_job_commit_pc(struct rknpu_job *job) +{ + struct rknpu_device *rknpu_dev = job->rknpu_dev; + struct rknpu_submit *args = job->args; + struct rknpu_gem_object *task_obj = + (struct rknpu_gem_object *)args->task_obj_addr; + struct rknpu_task *task_base = NULL; + struct rknpu_task *first_task = NULL; + struct rknpu_task *last_task = NULL; + int task_start = args->task_start; + int task_end = args->task_start + args->task_number - 1; + int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0; + + if (!task_obj) + return -EINVAL; + + if ((task_start + 1) * sizeof(*task_base) > task_obj->size || + (task_end + 1) * sizeof(*task_base) > task_obj->size) + return -EINVAL; + + task_base = task_obj->kv_addr; + + first_task = &task_base[task_start]; + last_task = &task_base[task_end]; + + REG_WRITE(first_task->regcmd_data, RKNPU_OFFSET_PC_DATA_ADDR); + + REG_WRITE(first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT - 1, + RKNPU_OFFSET_PC_DATA_AMOUNT); + + REG_WRITE(last_task->int_mask, RKNPU_OFFSET_INT_MASK); + + REG_WRITE(first_task->int_mask, RKNPU_OFFSET_INT_CLEAR); + + REG_WRITE(((0x6 | task_pp_en) << 12) | args->task_number, + RKNPU_OFFSET_PC_TASK_CONTROL); + + REG_WRITE(0x0, RKNPU_OFFSET_PC_DMA_BASE_ADDR); + + job->first_task = first_task; + job->last_task = last_task; + job->int_mask = last_task->int_mask; + + REG_WRITE(0x1, RKNPU_OFFSET_PC_OP_EN); + REG_WRITE(0x0, RKNPU_OFFSET_PC_OP_EN); + + return 0; +} + +static int rknpu_job_commit(struct rknpu_job *job) +{ + struct rknpu_device *rknpu_dev = job->rknpu_dev; + struct rknpu_submit *args = job->args; + + // switch to slave mode + REG_WRITE(0x1, RKNPU_OFFSET_PC_DATA_ADDR); + + if (!(args->flags & RKNPU_JOB_PC)) + return -EINVAL; + + return rknpu_job_commit_pc(job); +} + +static void rknpu_job_next(struct rknpu_device *rknpu_dev) +{ + struct rknpu_job *job = NULL; + unsigned long flags; + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + + if (rknpu_dev->job || list_empty(&rknpu_dev->todo_list)) { + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + return; + } + + job = list_first_entry(&rknpu_dev->todo_list, struct rknpu_job, head); + + list_del_init(&job->head); + + rknpu_dev->job = job; + + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + job->ret = rknpu_job_commit(job); +} + +static void rknpu_job_done(struct rknpu_job *job, int ret) +{ + struct rknpu_device *rknpu_dev = job->rknpu_dev; + unsigned long flags; + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + rknpu_dev->job = NULL; + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + job->flags |= RKNPU_JOB_DONE; + job->ret = ret; + + if (job->fence) + dma_fence_signal(job->fence); + + wake_up(&rknpu_dev->job_done_wq); + rknpu_job_next(rknpu_dev); + + if (job->flags & RKNPU_JOB_ASYNC) + schedule_work(&job->cleanup_work); +} + +static void rknpu_job_schedule(struct rknpu_job *job) +{ + struct rknpu_device *rknpu_dev = job->rknpu_dev; + unsigned long flags; + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + list_add_tail(&job->head, &rknpu_dev->todo_list); + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + rknpu_job_next(rknpu_dev); +} + +static void rknpu_job_abort(struct rknpu_job *job) +{ + struct rknpu_device *rknpu_dev = job->rknpu_dev; + unsigned long flags; + + msleep(100); + if (job->ret == -ETIMEDOUT) + rknpu_soft_reset(rknpu_dev); + + if (job == rknpu_dev->job) { + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + rknpu_dev->job = NULL; + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + } + + rknpu_job_cleanup(job); +} + +static inline uint32_t rknpu_fuzz_status(uint32_t status) +{ + uint32_t fuzz_status = 0; + + if ((status & 0x3) != 0) + fuzz_status |= 0x3; + + if ((status & 0xc) != 0) + fuzz_status |= 0xc; + + if ((status & 0x30) != 0) + fuzz_status |= 0x30; + + if ((status & 0xc0) != 0) + fuzz_status |= 0xc0; + + if ((status & 0x300) != 0) + fuzz_status |= 0x300; + + if ((status & 0xc00) != 0) + fuzz_status |= 0xc00; + + return fuzz_status; +} + +irqreturn_t rknpu_irq_handler(int irq, void *data) +{ + struct rknpu_device *rknpu_dev = data; + struct rknpu_job *job = rknpu_dev->job; + uint32_t status = 0; + + if (!job) + return IRQ_HANDLED; + + status = REG_READ(RKNPU_OFFSET_INT_STATUS); + REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR); + + job->int_status = status; + + if (rknpu_fuzz_status(status) != job->int_mask) { + LOG_DEBUG("irq: status = %#x, mask = %#x\n", status, + job->int_mask); + return IRQ_HANDLED; + } + + rknpu_job_done(job, 0); + + return IRQ_HANDLED; +} + +static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev) +{ + struct rknpu_job *job = NULL; + unsigned long flags; + ktime_t now = ktime_get(); + + job = rknpu_dev->job; + if (job && + ktime_to_ms(ktime_sub(now, job->timestamp)) >= job->args->timeout) { + rknpu_soft_reset(rknpu_dev); + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + rknpu_dev->job = NULL; + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + + do { + schedule_work(&job->cleanup_work); + + spin_lock_irqsave(&rknpu_dev->irq_lock, flags); + + if (!list_empty(&rknpu_dev->todo_list)) { + job = list_first_entry(&rknpu_dev->todo_list, + struct rknpu_job, head); + list_del_init(&job->head); + } else { + job = NULL; + } + + spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); + } while (job); + } +} + +int rknpu_submit_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); + struct rknpu_submit *args = data; + struct rknpu_job *job = NULL; + int ret = -EINVAL; + + if (args->task_number == 0) { + LOG_ERROR("invalid rknpu task number!\n"); + return -EINVAL; + } + + job = rknpu_job_alloc(rknpu_dev, args); + if (!job) { + LOG_ERROR("failed to allocate rknpu job!\n"); + return -ENOMEM; + } + + if (args->flags & RKNPU_JOB_FENCE) { + ret = rknpu_fence_alloc(job); + if (ret) { + rknpu_job_free(job); + return ret; + } + job->args->fence_fd = rknpu_fence_get_fd(job); + args->fence_fd = job->args->fence_fd; + } + + if (args->flags & RKNPU_JOB_NONBLOCK) { + job->flags |= RKNPU_JOB_ASYNC; + rknpu_job_timeout_clean(rknpu_dev); + rknpu_job_schedule(job); + ret = job->ret; + if (ret) { + rknpu_job_abort(job); + return ret; + } + } else { + rknpu_job_schedule(job); + if (args->flags & RKNPU_JOB_PC) + job->ret = rknpu_job_wait(job); + + args->task_counter = job->args->task_counter; + ret = job->ret; + if (!ret) + rknpu_job_cleanup(job); + else + rknpu_job_abort(job); + } + + return ret; +} + +int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version) +{ + if (version != NULL) + *version = REG_READ(RKNPU_OFFSET_VERSION); + + return 0; +} + +int rknpu_get_bw_priority(struct rknpu_device *rknpu_dev, uint32_t *priority, + uint32_t *expect, uint32_t *tw) +{ + void __iomem *base = rknpu_dev->bw_priority_base; + + if (!base) + return -EINVAL; + + spin_lock(&rknpu_dev->lock); + + if (priority != NULL) + *priority = _REG_READ(base, 0x0); + + if (expect != NULL) + *expect = _REG_READ(base, 0x8); + + if (tw != NULL) + *tw = _REG_READ(base, 0xc); + + spin_unlock(&rknpu_dev->lock); + + return 0; +} + +int rknpu_set_bw_priority(struct rknpu_device *rknpu_dev, uint32_t priority, + uint32_t expect, uint32_t tw) +{ + void __iomem *base = rknpu_dev->bw_priority_base; + + if (!base) + return -EINVAL; + + spin_lock(&rknpu_dev->lock); + + if (priority != 0) + _REG_WRITE(base, priority, 0x0); + + if (expect != 0) + _REG_WRITE(base, expect, 0x8); + + if (tw != 0) + _REG_WRITE(base, tw, 0xc); + + spin_unlock(&rknpu_dev->lock); + + return 0; +} + +int rknpu_clear_rw_amount(struct rknpu_device *rknpu_dev) +{ + spin_lock(&rknpu_dev->lock); + + REG_WRITE(0x80000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); + REG_WRITE(0x00000101, RKNPU_OFFSET_CLR_ALL_RW_AMOUNT); + + spin_unlock(&rknpu_dev->lock); + + return 0; +} + +int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr, + uint32_t *dt_rd, uint32_t *wd_rd) +{ + spin_lock(&rknpu_dev->lock); + + if (dt_wr != NULL) + *dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT); + + if (dt_rd != NULL) + *dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT); + + if (wd_rd != NULL) + *wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT); + + spin_unlock(&rknpu_dev->lock); + + return 0; +} + +int rknpu_get_total_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *amount) +{ + uint32_t dt_wr = 0; + uint32_t dt_rd = 0; + uint32_t wd_rd = 0; + int ret = -EINVAL; + + ret = rknpu_get_rw_amount(rknpu_dev, &dt_wr, &dt_rd, &wd_rd); + + if (amount != NULL) + *amount = dt_wr + dt_rd + wd_rd; + + return ret; +} diff --git a/drivers/rknpu/rknpu_reset.c b/drivers/rknpu/rknpu_reset.c new file mode 100644 index 000000000000..1b0f77df9d87 --- /dev/null +++ b/drivers/rknpu/rknpu_reset.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd + * Author: Felix Zeng + */ + +#include +#include + +#include "rknpu_reset.h" + +static inline struct reset_control *rknpu_reset_control_get(struct device *dev, + const char *name) +{ + struct reset_control *rst = NULL; + + rst = devm_reset_control_get(dev, name); + if (IS_ERR(rst)) + LOG_DEV_ERROR(dev, + "failed to get rknpu reset control: %s, %ld\n", + name, PTR_ERR(rst)); + + return rst; +} + +int rknpu_reset_get(struct rknpu_device *rknpu_dev) +{ + struct reset_control *srst_a = NULL; + struct reset_control *srst_h = NULL; + + srst_a = rknpu_reset_control_get(rknpu_dev->dev, "srst_a"); + if (IS_ERR(srst_a)) + return PTR_ERR(srst_a); + + rknpu_dev->srst_a = srst_a; + + srst_h = devm_reset_control_get(rknpu_dev->dev, "srst_h"); + if (IS_ERR(srst_h)) + return PTR_ERR(srst_h); + + rknpu_dev->srst_h = srst_h; + + return 0; +} + +static int rknpu_reset_assert(struct reset_control *rst) +{ + int ret = -EINVAL; + + if (!rst) + return -EINVAL; + + ret = reset_control_assert(rst); + if (ret < 0) { + LOG_ERROR("failed to assert rknpu reset: %d\n", ret); + return ret; + } + + return 0; +} + +static int rknpu_reset_deassert(struct reset_control *rst) +{ + int ret = -EINVAL; + + if (!rst) + return -EINVAL; + + ret = reset_control_deassert(rst); + if (ret < 0) { + LOG_ERROR("failed to deassert rknpu reset: %d\n", ret); + return ret; + } + + return 0; +} + +int rknpu_soft_reset(struct rknpu_device *rknpu_dev) +{ + struct iommu_domain *domain = NULL; + int ret = -EINVAL; + + if (rknpu_dev->bypass_soft_reset) { + LOG_WARN("bypass soft reset\n"); + return 0; + } + + LOG_INFO("soft reset\n"); + + ret = rknpu_reset_assert(rknpu_dev->srst_a); + ret |= rknpu_reset_assert(rknpu_dev->srst_h); + + udelay(10); + + ret |= rknpu_reset_deassert(rknpu_dev->srst_a); + ret |= rknpu_reset_deassert(rknpu_dev->srst_h); + + if (ret) { + LOG_DEV_ERROR(rknpu_dev->dev, + "failed to soft reset for rknpu: %d\n", ret); + return ret; + } + + if (rknpu_dev->iommu_en) + domain = iommu_get_domain_for_dev(rknpu_dev->dev); + + if (domain) { + iommu_detach_device(domain, rknpu_dev->dev); + iommu_attach_device(domain, rknpu_dev->dev); + } + + return ret; +}