shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-07 11:26:02 +09:00
Code Issues Packages Projects Releases Wiki Activity

driver: rknpu: Update rknpu driver, version: 0.7.0

Browse Source 
* Add support for rv1106/rv1103

Signed-off-by: Felix Zeng <felix.zeng@rock-chips.com>
Change-Id: Ic681c8657cf372adbdc39c30284d66d736030775
This commit is contained in:
Felix Zeng
2022-03-15 16:44:00 +08:00
committed by Tao Huang
parent 0bb9818fe4
commit 1e714a44c9
12 changed files with 1706 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
drivers/rknpu/Kconfig
View File

@@ -1,10 +1,54 @@
# SPDX-License-Identifier: GPL-2.0
menu "RKNPU"
depends on ARCH_ROCKCHIP && DRM
depends on ARCH_ROCKCHIP
config ROCKCHIP_RKNPU
tristate "ROCKCHIP_RKNPU"
depends on DRM || DMABUF_HEAPS_ROCKCHIP_CMA_HEAP
help
rknpu module.
if ROCKCHIP_RKNPU
config ROCKCHIP_RKNPU_DEBUG_FS
bool "RKNPU debugfs"
depends on DEBUG_FS
default y
help
Enable debugfs to debug RKNPU usage.
config ROCKCHIP_RKNPU_PROC_FS
bool "RKNPU procfs"
depends on PROC_FS
help
Enable procfs to debug RKNPU usage.
config ROCKCHIP_RKNPU_FENCE
bool "RKNPU fence"
depends on SYNC_FILE
help
Enable fence support for RKNPU.
choice
prompt "RKNPU memory manager"
default ROCKCHIP_RKNPU_DRM_GEM
help
Select RKNPU memory manager
config ROCKCHIP_RKNPU_DRM_GEM
bool "RKNPU DRM GEM"
depends on DRM
help
Enable RKNPU memory manager by DRM GEM.
config ROCKCHIP_RKNPU_DMA_HEAP
bool "RKNPU DMA heap"
depends on DMABUF_HEAPS_ROCKCHIP_CMA_HEAP
help
Enable RKNPU memory manager by DMA Heap.
endchoice
endif
endmenu

7
drivers/rknpu/Makefile
View File

@@ -2,10 +2,13 @@
obj-$(CONFIG_ROCKCHIP_RKNPU) += rknpu.o
ccflags-y += -I$(srctree)/$(src)/include
ccflags-y += -I$(src)/include
ccflags-y += -Werror
rknpu-y += rknpu_drv.o
rknpu-y += rknpu_reset.o
rknpu-y += rknpu_job.o
rknpu-y += rknpu_gem.o
rknpu-y += rknpu_fence.o
rknpu-y += rknpu_debugger.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_FENCE) += rknpu_fence.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_DRM_GEM) += rknpu_gem.o
rknpu-$(CONFIG_ROCKCHIP_RKNPU_DMA_HEAP) += rknpu_mem.o

86
drivers/rknpu/include/rknpu_debugger.h Normal file
View File

@@ -0,0 +1,86 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
* Author: Felix Zeng <felix.zeng@rock-chips.com>
*/
#ifndef __LINUX_RKNPU_DEBUGGER_H_
#define __LINUX_RKNPU_DEBUGGER_H_
/*
* struct rknpu_debugger - rknpu debugger information
*
* This structure represents a debugger to be created by the rknpu driver
* or core.
*/
struct rknpu_debugger {
#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
/* Directory of debugfs file */
struct dentry *debugfs_dir;
struct list_head debugfs_entry_list;
struct mutex debugfs_lock;
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
/* Directory of procfs file */
struct proc_dir_entry *procfs_dir;
struct list_head procfs_entry_list;
struct mutex procfs_lock;
#endif
};
/*
* struct rknpu_debugger_list - debugfs/procfs info list entry
*
* This structure represents a debugfs/procfs file to be created by the npu
* driver or core.
*/
struct rknpu_debugger_list {
/* File name */
const char *name;
/*
* Show callback. &seq_file->private will be set to the &struct
* rknpu_debugger_node corresponding to the instance of this info
* on a given &struct rknpu_debugger.
*/
int (*show)(struct seq_file *seq, void *data);
/*
* Write callback. &seq_file->private will be set to the &struct
* rknpu_debugger_node corresponding to the instance of this info
* on a given &struct rknpu_debugger.
*/
ssize_t (*write)(struct file *file, const char __user *ubuf,
size_t len, loff_t *offp);
/* Procfs/Debugfs private data. */
void *data;
};
/*
* struct rknpu_debugger_node - Nodes for debugfs/procfs
*
* This structure represents each instance of procfs/debugfs created from the
* template.
*/
struct rknpu_debugger_node {
struct rknpu_debugger *debugger;
/* template for this node. */
const struct rknpu_debugger_list *info_ent;
/* Each Procfs/Debugfs file. */
#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
struct dentry *dent;
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
struct proc_dir_entry *pent;
#endif
struct list_head list;
};
int rknpu_debugger_init(struct rknpu_device *rknpu_dev);
int rknpu_debugger_remove(struct rknpu_device *rknpu_dev);
#endif /* __LINUX_RKNPU_FENCE_H_ */

51
drivers/rknpu/include/rknpu_drv.h
View File

@@ -14,33 +14,43 @@
#include <linux/spinlock.h>
#include <linux/regulator/consumer.h>
#include <linux/version.h>
#include <linux/hrtimer.h>
#include <linux/miscdevice.h>
#ifndef FPGA_PLATFORM
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
#include <soc/rockchip/rockchip_opp_select.h>
#endif
#endif
#include "rknpu_job.h"
#include "rknpu_fence.h"
#include "rknpu_debugger.h"
#define DRIVER_NAME "rknpu"
#define DRIVER_DESC "RKNPU driver"
#define DRIVER_DATE "20211227"
#define DRIVER_DATE "20220328"
#define DRIVER_MAJOR 0
#define DRIVER_MINOR 6
#define DRIVER_PATCHLEVEL 4
#define DRIVER_MINOR 7
#define DRIVER_PATCHLEVEL 0
#define LOG_TAG "RKNPU"
/* sample interval: 1000ms */
#define RKNPU_LOAD_INTERVAL 1000000000
#define LOG_INFO(fmt, args...) pr_info(LOG_TAG ": " fmt, ##args)
#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE
#define LOG_WARN(fmt, args...) pr_warn(LOG_TAG ": " fmt, ##args)
#else
#define LOG_WARN(fmt, args...) pr_warning(LOG_TAG ": " fmt, ##args)
#endif
#define LOG_DEBUG(fmt, args...) DRM_DEBUG_DRIVER(LOG_TAG ": " fmt, ##args)
#define LOG_DEBUG(fmt, args...) pr_devel(LOG_TAG ": " fmt, ##args)
#define LOG_ERROR(fmt, args...) pr_err(LOG_TAG ": " fmt, ##args)
#define LOG_DEV_INFO(dev, fmt, args...) dev_info(dev, LOG_TAG ": " fmt, ##args)
#define LOG_DEV_WARN(dev, fmt, args...) dev_warn(dev, LOG_TAG ": " fmt, ##args)
#define LOG_DEV_DEBUG(dev, fmt, args...) \
DRM_DEV_DEBUG_DRIVER(dev, LOG_TAG ": " fmt, ##args)
#define LOG_DEV_DEBUG(dev, fmt, args...) dev_dbg(dev, LOG_TAG ": " fmt, ##args)
#define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args)
struct npu_reset_data {
@@ -52,7 +62,9 @@ struct rknpu_config {
__u32 bw_priority_addr;
__u32 bw_priority_length;
__u64 dma_mask;
__u32 pc_data_extra_amount;
__u32 pc_data_amount_scale;
__u32 pc_task_number_bits;
__u32 pc_task_number_mask;
__u32 bw_enable;
const struct npu_irqs_data *irqs;
const struct npu_reset_data *resets;
@@ -60,11 +72,17 @@ struct rknpu_config {
int num_resets;
};
struct rknpu_timer {
__u32 busy_time;
__u32 busy_time_record;
};
struct rknpu_subcore_data {
struct list_head todo_list;
wait_queue_head_t job_done_wq;
struct rknpu_job *job;
uint64_t task_num;
struct rknpu_timer timer;
};
/**
@@ -77,11 +95,17 @@ struct rknpu_subcore_data {
struct rknpu_device {
void __iomem *base[RKNPU_MAX_CORES];
struct device *dev;
struct device *fake_dev;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct drm_device *drm_dev;
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
struct miscdevice miscdev;
struct rk_dma_heap *heap;
#endif
atomic_t sequence;
spinlock_t lock;
spinlock_t irq_lock;
struct mutex power_lock;
struct rknpu_subcore_data subcore_datas[RKNPU_MAX_CORES];
const struct rknpu_config *config;
void __iomem *bw_priority_base;
@@ -97,7 +121,11 @@ struct rknpu_device {
struct ipa_power_model_data *model_data;
struct thermal_cooling_device *devfreq_cooling;
struct devfreq *devfreq;
#ifndef FPGA_PLATFORM
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
struct rockchip_opp_info opp_info;
#endif
#endif
unsigned long current_freq;
unsigned long current_volt;
int bypass_irq_handler;
@@ -106,6 +134,13 @@ struct rknpu_device {
struct device *genpd_dev_npu1;
struct device *genpd_dev_npu2;
bool multiple_domains;
atomic_t power_refcount;
struct delayed_work power_off_work;
struct workqueue_struct *power_off_wq;
bool is_powered;
struct rknpu_debugger debugger;
struct hrtimer timer;
ktime_t kt;
};
#endif /* __LINUX_RKNPU_DRV_H_ */

34
drivers/rknpu/include/rknpu_ioctl.h
View File

@@ -14,6 +14,10 @@
#define __user
#endif
#ifndef __packed
#define __packed __attribute__((packed))
#endif
#define RKNPU_OFFSET_VERSION 0x0
#define RKNPU_OFFSET_PC_OP_EN 0x8
#define RKNPU_OFFSET_PC_DATA_ADDR 0x10
@@ -36,6 +40,8 @@
#define RKNPU_INT_CLEAR 0x1ffff
#define RKNPU_PC_DATA_EXTRA_AMOUNT 4
#define RKNPU_STR_HELPER(x) #x
#define RKNPU_GET_DRV_VERSION_STRING(MAJOR, MINOR, PATCHLEVEL) \
@@ -119,6 +125,8 @@ enum e_rknpu_action {
RKNPU_GET_TOTAL_RW_AMOUNT = 17,
RKNPU_GET_IOMMU_EN = 18,
RKNPU_SET_PROC_NICE = 19,
RKNPU_POWER_ON = 20,
RKNPU_POWER_OFF = 21,
};
/**
@@ -157,10 +165,12 @@ struct rknpu_mem_map {
*
* @handle: handle of the buffer.
* @reserved: reserved for padding.
* @obj_addr: rknpu_mem_object addr.
*/
struct rknpu_mem_destroy {
__u32 handle;
__u32 reserved;
__u64 obj_addr;
};
/**
@@ -192,7 +202,7 @@ struct rknpu_mem_sync {
* @int_status: interrupt status
* @regcfg_amount: register config number
* @regcfg_offset: offset for register config
* @regcmd_data: data for register command
* @regcmd_addr: address for register command
*
*/
struct rknpu_task {
@@ -204,7 +214,7 @@ struct rknpu_task {
__u32 int_status;
__u32 regcfg_amount;
__u32 regcfg_offset;
__u64 regcmd_data;
__u64 regcmd_addr;
} __packed;
/**
@@ -217,7 +227,6 @@ struct rknpu_task {
struct rknpu_subcore_task {
__u32 task_start;
__u32 task_number;
__u32 task_end;
};
/**
@@ -231,10 +240,11 @@ struct rknpu_subcore_task {
* @priority: submit priority
* @task_obj_addr: address of task object
* @regcfg_obj_addr: address of register config object
* @task_base_addr: task base address
* @user_data: (optional) user data
* @sequence: submit sequence
* @core_mask: core mask of rknpu
* @fence_fd: dma fence fd
* @subcore_task: subcore task
*
*/
struct rknpu_submit {
@@ -246,8 +256,8 @@ struct rknpu_submit {
__s32 priority;
__u64 task_obj_addr;
__u64 regcfg_obj_addr;
__u64 task_base_addr;
__u64 user_data;
__u64 sequence;
__u32 core_mask;
__s32 fence_fd;
struct rknpu_subcore_task subcore_task[5];
@@ -272,6 +282,11 @@ struct rknpu_action {
#define RKNPU_MEM_DESTROY 0x04
#define RKNPU_MEM_SYNC 0x05
#define RKNPU_IOC_MAGIC 'r'
#define RKNPU_IOW(nr, type) _IOW(RKNPU_IOC_MAGIC, nr, type)
#define RKNPU_IOR(nr, type) _IOR(RKNPU_IOC_MAGIC, nr, type)
#define RKNPU_IOWR(nr, type) _IOWR(RKNPU_IOC_MAGIC, nr, type)
#if defined(__arm__) || defined(__aarch64__)
#include <drm/drm.h>
@@ -289,6 +304,15 @@ struct rknpu_action {
#define DRM_IOCTL_RKNPU_MEM_SYNC \
DRM_IOWR(DRM_COMMAND_BASE + RKNPU_MEM_SYNC, struct rknpu_mem_sync)
#define IOCTL_RKNPU_ACTION RKNPU_IOWR(RKNPU_ACTION, struct rknpu_action)
#define IOCTL_RKNPU_SUBMIT RKNPU_IOWR(RKNPU_SUBMIT, struct rknpu_submit)
#define IOCTL_RKNPU_MEM_CREATE \
RKNPU_IOWR(RKNPU_MEM_CREATE, struct rknpu_mem_create)
#define IOCTL_RKNPU_MEM_MAP RKNPU_IOWR(RKNPU_MEM_MAP, struct rknpu_mem_map)
#define IOCTL_RKNPU_MEM_DESTROY \
RKNPU_IOWR(RKNPU_MEM_DESTROY, struct rknpu_mem_destroy)
#define IOCTL_RKNPU_MEM_SYNC RKNPU_IOWR(RKNPU_MEM_SYNC, struct rknpu_mem_sync)
#endif
#endif

7
drivers/rknpu/include/rknpu_job.h
View File

@@ -9,6 +9,7 @@
#include <linux/spinlock.h>
#include <linux/dma-fence.h>
#include <linux/irq.h>
#include <drm/drm_device.h>
@@ -41,14 +42,20 @@ struct rknpu_job {
uint32_t use_core_num;
uint32_t run_count;
uint32_t interrupt_count;
ktime_t hw_recoder_time;
};
irqreturn_t rknpu_core0_irq_handler(int irq, void *data);
irqreturn_t rknpu_core1_irq_handler(int irq, void *data);
irqreturn_t rknpu_core2_irq_handler(int irq, void *data);
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
int rknpu_submit_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
int rknpu_submit_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
#endif
int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version);

43
drivers/rknpu/include/rknpu_mem.h Normal file
View File

@@ -0,0 +1,43 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
* Author: Felix Zeng <felix.zeng@rock-chips.com>
*/
#ifndef __LINUX_RKNPU_MEM_H
#define __LINUX_RKNPU_MEM_H
#include <linux/mm_types.h>
#include <linux/version.h>
/*
* rknpu DMA buffer structure.
*
* @flags: indicate memory type to allocated buffer and cache attribute.
* @size: size requested from user, in bytes and this size is aligned
* in page unit.
* @kv_addr: kernel virtual address to allocated memory region.
* @dma_addr: bus address(accessed by dma) to allocated memory region.
* - this address could be physical address without IOMMU and
* device address with IOMMU.
* @pages: Array of backing pages.
* @sgt: Imported sg_table.
* @dmabuf: buffer for this attachment.
* @owner: Is this memory internally allocated.
*/
struct rknpu_mem_object {
unsigned long flags;
unsigned long size;
void __iomem *kv_addr;
dma_addr_t dma_addr;
struct page **pages;
struct sg_table *sgt;
struct dma_buf *dmabuf;
unsigned int owner;
};
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
#endif

354
drivers/rknpu/rknpu_debugger.c Normal file
View File

@@ -0,0 +1,354 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
* Author: Felix Zeng <felix.zeng@rock-chips.com>
*/
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/syscalls.h>
#include <linux/debugfs.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <asm/div64.h>
#include "rknpu_drv.h"
#include "rknpu_debugger.h"
#define RKNPU_DEBUGGER_ROOT_NAME "rknpu"
static int rknpu_version_show(struct seq_file *m, void *data)
{
seq_printf(m, "%s: v%d.%d.%d\n", DRIVER_DESC, DRIVER_MAJOR,
DRIVER_MINOR, DRIVER_PATCHLEVEL);
return 0;
}
static int rknpu_load_show(struct seq_file *m, void *data)
{
struct rknpu_debugger_node *node = m->private;
struct rknpu_debugger *debugger = node->debugger;
struct rknpu_device *rknpu_dev =
container_of(debugger, struct rknpu_device, debugger);
struct rknpu_subcore_data *subcore_data = NULL;
unsigned long flags;
int i;
int load;
uint64_t busy_time_total, div_value;
seq_puts(m, "NPU load: ");
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
subcore_data = &rknpu_dev->subcore_datas[i];
if (rknpu_dev->config->num_irqs > 1)
seq_printf(m, " Core%d: ", i);
spin_lock_irqsave(&rknpu_dev->irq_lock, flags);
busy_time_total = subcore_data->timer.busy_time_record;
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
div_value = (RKNPU_LOAD_INTERVAL / 100000);
do_div(busy_time_total, div_value);
load = busy_time_total;
if (rknpu_dev->config->num_irqs > 1)
seq_printf(m, "%2.d%%,", load);
else
seq_printf(m, "%2.d%%", load);
}
seq_puts(m, "\n");
return 0;
}
struct rknpu_debugger_list rknpu_debugger_root_list[] = {
{ "driver_version", rknpu_version_show, NULL, NULL },
{ "load", rknpu_load_show, NULL, NULL },
};
static ssize_t rknpu_debugger_write(struct file *file, const char __user *ubuf,
size_t len, loff_t *offp)
{
struct seq_file *priv = file->private_data;
struct rknpu_debugger_node *node = priv->private;
if (node->info_ent->write)
return node->info_ent->write(file, ubuf, len, offp);
else
return len;
}
static int rknpu_debugfs_open(struct inode *inode, struct file *file)
{
struct rknpu_debugger_node *node = inode->i_private;
return single_open(file, node->info_ent->show, node);
}
static const struct file_operations rknpu_debugfs_fops = {
.owner = THIS_MODULE,
.open = rknpu_debugfs_open,
.read = seq_read,
.llseek = seq_lseek,
.release = single_release,
.write = rknpu_debugger_write,
};
#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
static int rknpu_debugfs_remove_files(struct rknpu_debugger *debugger)
{
struct rknpu_debugger_node *pos, *q;
struct list_head *entry_list;
mutex_lock(&debugger->debugfs_lock);
/* Delete debugfs entry list */
entry_list = &debugger->debugfs_entry_list;
list_for_each_entry_safe(pos, q, entry_list, list) {
if (pos->dent == NULL)
continue;
list_del(&pos->list);
kfree(pos);
pos = NULL;
}
/* Delete all debugfs node in this directory */
debugfs_remove_recursive(debugger->debugfs_dir);
debugger->debugfs_dir = NULL;
mutex_unlock(&debugger->debugfs_lock);
return 0;
}
static int rknpu_debugfs_create_files(const struct rknpu_debugger_list *files,
int count, struct dentry *root,
struct rknpu_debugger *debugger)
{
int i;
struct dentry *ent;
struct rknpu_debugger_node *tmp;
for (i = 0; i < count; i++) {
tmp = kmalloc(sizeof(struct rknpu_debugger_node), GFP_KERNEL);
if (tmp == NULL) {
LOG_ERROR(
"Cannot alloc node path /sys/kernel/debug/%pd/%s\n",
root, files[i].name);
goto MALLOC_FAIL;
}
tmp->info_ent = &files[i];
tmp->debugger = debugger;
ent = debugfs_create_file(files[i].name, S_IFREG | S_IRUGO,
root, tmp, &rknpu_debugfs_fops);
if (!ent) {
LOG_ERROR("Cannot create /sys/kernel/debug/%pd/%s\n",
root, files[i].name);
goto CREATE_FAIL;
}
tmp->dent = ent;
mutex_lock(&debugger->debugfs_lock);
list_add_tail(&tmp->list, &debugger->debugfs_entry_list);
mutex_unlock(&debugger->debugfs_lock);
}
return 0;
CREATE_FAIL:
kfree(tmp);
MALLOC_FAIL:
rknpu_debugfs_remove_files(debugger);
return -1;
}
static int rknpu_debugfs_remove(struct rknpu_debugger *debugger)
{
rknpu_debugfs_remove_files(debugger);
return 0;
}
static int rknpu_debugfs_init(struct rknpu_debugger *debugger)
{
int ret;
debugger->debugfs_dir =
debugfs_create_dir(RKNPU_DEBUGGER_ROOT_NAME, NULL);
if (IS_ERR_OR_NULL(debugger->debugfs_dir)) {
LOG_ERROR("failed on mkdir /sys/kernel/debug/%s\n",
RKNPU_DEBUGGER_ROOT_NAME);
debugger->debugfs_dir = NULL;
return -EIO;
}
ret = rknpu_debugfs_create_files(rknpu_debugger_root_list,
ARRAY_SIZE(rknpu_debugger_root_list),
debugger->debugfs_dir, debugger);
if (ret) {
LOG_ERROR(
"Could not install rknpu_debugger_root_list debugfs\n");
goto CREATE_FAIL;
}
return 0;
CREATE_FAIL:
rknpu_debugfs_remove(debugger);
return ret;
}
#endif /* #ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS */
#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
static int rknpu_procfs_open(struct inode *inode, struct file *file)
{
struct rknpu_debugger_node *node = PDE_DATA(inode);
return single_open(file, node->info_ent->show, node);
}
static const struct proc_ops rknpu_procfs_fops = {
.proc_open = rknpu_procfs_open,
.proc_read = seq_read,
.proc_lseek = seq_lseek,
.proc_release = single_release,
.proc_write = rknpu_debugger_write,
};
static int rknpu_procfs_remove_files(struct rknpu_debugger *debugger)
{
struct rknpu_debugger_node *pos, *q;
struct list_head *entry_list;
mutex_lock(&debugger->procfs_lock);
/* Delete procfs entry list */
entry_list = &debugger->procfs_entry_list;
list_for_each_entry_safe(pos, q, entry_list, list) {
if (pos->pent == NULL)
continue;
list_del(&pos->list);
kfree(pos);
pos = NULL;
}
/* Delete all procfs node in this directory */
proc_remove(debugger->procfs_dir);
debugger->procfs_dir = NULL;
mutex_unlock(&debugger->procfs_lock);
return 0;
}
static int rknpu_procfs_create_files(const struct rknpu_debugger_list *files,
int count, struct proc_dir_entry *root,
struct rknpu_debugger *debugger)
{
int i;
struct proc_dir_entry *ent;
struct rknpu_debugger_node *tmp;
for (i = 0; i < count; i++) {
tmp = kmalloc(sizeof(struct rknpu_debugger_node), GFP_KERNEL);
if (tmp == NULL) {
LOG_ERROR("Cannot alloc node path for /proc/%s/%s\n",
RKNPU_DEBUGGER_ROOT_NAME, files[i].name);
goto MALLOC_FAIL;
}
tmp->info_ent = &files[i];
tmp->debugger = debugger;
ent = proc_create_data(files[i].name, S_IFREG | S_IRUGO, root,
&rknpu_procfs_fops, tmp);
if (!ent) {
LOG_ERROR("Cannot create /proc/%s/%s\n",
RKNPU_DEBUGGER_ROOT_NAME, files[i].name);
goto CREATE_FAIL;
}
tmp->pent = ent;
mutex_lock(&debugger->procfs_lock);
list_add_tail(&tmp->list, &debugger->procfs_entry_list);
mutex_unlock(&debugger->procfs_lock);
}
return 0;
CREATE_FAIL:
kfree(tmp);
MALLOC_FAIL:
rknpu_procfs_remove_files(debugger);
return -1;
}
int rknpu_procfs_remove(struct rknpu_debugger *debugger)
{
rknpu_procfs_remove_files(debugger);
return 0;
}
int rknpu_procfs_init(struct rknpu_debugger *debugger)
{
int ret;
debugger->procfs_dir = proc_mkdir(RKNPU_DEBUGGER_ROOT_NAME, NULL);
if (IS_ERR_OR_NULL(debugger->procfs_dir)) {
pr_err("failed on mkdir /proc/%s\n", RKNPU_DEBUGGER_ROOT_NAME);
debugger->procfs_dir = NULL;
return -EIO;
}
ret = rknpu_procfs_create_files(rknpu_debugger_root_list,
ARRAY_SIZE(rknpu_debugger_root_list),
debugger->procfs_dir, debugger);
if (ret) {
pr_err("Could not install rknpu_debugger_root_list procfs\n");
goto CREATE_FAIL;
}
return 0;
CREATE_FAIL:
rknpu_procfs_remove(debugger);
return ret;
}
#endif /* #ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS */
int rknpu_debugger_init(struct rknpu_device *rknpu_dev)
{
#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
mutex_init(&rknpu_dev->debugger.debugfs_lock);
INIT_LIST_HEAD(&rknpu_dev->debugger.debugfs_entry_list);
rknpu_debugfs_init(&rknpu_dev->debugger);
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
mutex_init(&rknpu_dev->debugger.procfs_lock);
INIT_LIST_HEAD(&rknpu_dev->debugger.procfs_entry_list);
rknpu_procfs_init(&rknpu_dev->debugger);
#endif
return 0;
}
int rknpu_debugger_remove(struct rknpu_device *rknpu_dev)
{
#ifdef CONFIG_ROCKCHIP_RKNPU_DEBUG_FS
rknpu_debugfs_remove(&rknpu_dev->debugger);
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_PROC_FS
rknpu_procfs_remove(&rknpu_dev->debugger);
#endif
return 0;
}

796
drivers/rknpu/rknpu_drv.c
View File

File diff suppressed because it is too large Load Diff

67
drivers/rknpu/rknpu_gem.c
View File

@@ -21,13 +21,14 @@
#include "rknpu_ioctl.h"
#include "rknpu_gem.h"
#define RKNPU_GEM_ALLOC_FROM_PAGES 0
#define RKNPU_GEM_ALLOC_FROM_PAGES 1
#if RKNPU_GEM_ALLOC_FROM_PAGES
static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj)
{
struct drm_device *drm = rknpu_obj->base.dev;
struct scatterlist *s = NULL;
dma_addr_t dma_addr = 0;
int ret = -EINVAL, i = 0;
rknpu_obj->pages = drm_gem_get_pages(&rknpu_obj->base);
@@ -48,35 +49,44 @@ static int rknpu_gem_get_pages(struct rknpu_gem_object *rknpu_obj)
goto put_pages;
}
for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
sg_dma_address(s) = sg_phys(s);
LOG_DEBUG(
"gem pages alloc sgt[%d], phys_address: %#llx, length: %#x\n",
i, (__u64)s->dma_address, s->length);
}
ret = dma_map_sg_attrs(drm->dev, rknpu_obj->sgt->sgl,
rknpu_obj->sgt->nents, DMA_BIDIRECTIONAL,
rknpu_obj->dma_attrs);
ret = dma_map_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
DMA_BIDIRECTIONAL);
if (ret == 0) {
LOG_DEV_ERROR(drm->dev, "failed to map sg table.\n");
ret = -EFAULT;
goto free_sgt;
}
dma_sync_sg_for_device(drm->dev, rknpu_obj->sgt->sgl,
rknpu_obj->sgt->nents, DMA_TO_DEVICE);
if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING) {
rknpu_obj->kv_addr =
vmap(rknpu_obj->pages, rknpu_obj->num_pages, VM_MAP,
PAGE_KERNEL);
rknpu_obj->cookie = vmap(rknpu_obj->pages, rknpu_obj->num_pages,
VM_MAP, PAGE_KERNEL);
if (!rknpu_obj->cookie)
goto unmap_sg;
rknpu_obj->kv_addr = rknpu_obj->cookie;
}
rknpu_obj->dma_addr = (__u64)sg_dma_address(rknpu_obj->sgt->sgl);
dma_addr = sg_dma_address(rknpu_obj->sgt->sgl);
rknpu_obj->dma_addr = dma_addr;
for_each_sg(rknpu_obj->sgt->sgl, s, rknpu_obj->sgt->nents, i) {
dma_addr += s->length;
LOG_DEBUG(
"gem pages alloc sgt[%d], dma_address: %#llx, length: %#x\n",
i, (__u64)dma_addr, s->length);
}
return 0;
unmap_sg:
dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
DMA_BIDIRECTIONAL);
free_sgt:
sg_free_table(rknpu_obj->sgt);
kfree(rknpu_obj->sgt);
put_pages:
drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, false, false);
@@ -90,8 +100,9 @@ static void rknpu_gem_put_pages(struct rknpu_gem_object *rknpu_obj)
if (rknpu_obj->flags & RKNPU_MEM_KERNEL_MAPPING)
vunmap(rknpu_obj->kv_addr);
dma_map_sg_attrs(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
DMA_BIDIRECTIONAL, rknpu_obj->dma_attrs);
dma_unmap_sg(drm->dev, rknpu_obj->sgt->sgl, rknpu_obj->sgt->nents,
DMA_BIDIRECTIONAL);
drm_gem_put_pages(&rknpu_obj->base, rknpu_obj->pages, true, true);
sg_free_table(rknpu_obj->sgt);
kfree(rknpu_obj->sgt);
@@ -911,9 +922,8 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
struct rknpu_gem_object *rknpu_obj = NULL;
struct rknpu_mem_sync *args = data;
struct scatterlist *sg;
dma_addr_t sg_dma_addr;
unsigned long length, offset = 0;
unsigned long sg_offset, sg_left, size = 0;
unsigned long sg_left, size = 0;
unsigned long len = 0;
int i;
@@ -937,11 +947,6 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
DMA_FROM_DEVICE);
}
} else {
struct drm_device *drm = rknpu_obj->base.dev;
struct rknpu_device *rknpu_dev = drm->dev_private;
WARN_ON(!rknpu_dev->fake_dev);
length = args->size;
offset = args->offset;
@@ -951,21 +956,17 @@ int rknpu_gem_sync_ioctl(struct drm_device *dev, void *data,
if (len <= offset)
continue;
sg_dma_addr = sg_dma_address(sg);
sg_left = len - offset;
sg_offset = sg->length - sg_left;
size = (length < sg_left) ? length : sg_left;
if (args->flags & RKNPU_MEM_SYNC_TO_DEVICE) {
dma_sync_single_range_for_device(
rknpu_dev->fake_dev, sg_dma_addr,
sg_offset, size, DMA_TO_DEVICE);
dma_sync_sg_for_device(dev->dev, sg, 1,
DMA_TO_DEVICE);
}
if (args->flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
dma_sync_single_range_for_cpu(
rknpu_dev->fake_dev, sg_dma_addr,
sg_offset, size, DMA_FROM_DEVICE);
dma_sync_sg_for_cpu(dev->dev, sg, 1,
DMA_FROM_DEVICE);
}
offset += size;

250
drivers/rknpu/rknpu_job.c
View File

@@ -15,6 +15,7 @@
#include "rknpu_gem.h"
#include "rknpu_fence.h"
#include "rknpu_job.h"
#include "rknpu_mem.h"
#define _REG_READ(base, offset) readl(base + (offset))
#define _REG_WRITE(base, value, offset) writel(value, base + (offset))
@@ -57,15 +58,17 @@ static int rknn_get_task_number(struct rknpu_job *job, int core_index)
static void rknpu_job_free(struct rknpu_job *job)
{
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct rknpu_gem_object *task_obj = NULL;
if (job->fence)
dma_fence_put(job->fence);
task_obj =
(struct rknpu_gem_object *)(uintptr_t)job->args->task_obj_addr;
if (task_obj)
rknpu_gem_object_put(&task_obj->base);
#endif
if (job->fence)
dma_fence_put(job->fence);
if (job->args_owner)
kfree(job->args);
@@ -92,8 +95,9 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
struct rknpu_submit *args)
{
struct rknpu_job *job = NULL;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct rknpu_gem_object *task_obj = NULL;
#endif
if (rknpu_dev->config->num_irqs == 1)
args->core_mask = RKNPU_CORE0_MASK;
@@ -108,10 +112,11 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev,
((args->core_mask & RKNPU_CORE2_MASK) >> 2);
job->run_count = job->use_core_num;
job->interrupt_count = job->use_core_num;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
task_obj = (struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
if (task_obj)
rknpu_gem_object_get(&task_obj->base);
#endif
if (!(args->flags & RKNPU_JOB_NONBLOCK)) {
job->args = args;
@@ -159,7 +164,9 @@ static inline int rknpu_job_wait(struct rknpu_job *job)
if (args->flags & RKNPU_JOB_PC) {
uint32_t task_status =
REG_READ(RKNPU_OFFSET_PC_TASK_STATUS);
args->task_counter = (task_status & 0xfff);
args->task_counter =
(task_status &
rknpu_dev->config->pc_task_number_mask);
}
return ret < 0 ? ret : -ETIMEDOUT;
}
@@ -173,8 +180,14 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
{
struct rknpu_device *rknpu_dev = job->rknpu_dev;
struct rknpu_submit *args = job->args;
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
struct rknpu_gem_object *task_obj =
(struct rknpu_gem_object *)(uintptr_t)args->task_obj_addr;
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
struct rknpu_mem_object *task_obj =
(struct rknpu_mem_object *)(uintptr_t)args->task_obj_addr;
#endif
struct rknpu_task *task_base = NULL;
struct rknpu_task *first_task = NULL;
struct rknpu_task *last_task = NULL;
@@ -183,33 +196,45 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
int task_end = args->task_start + args->task_number - 1;
int task_number = args->task_number;
int task_pp_en = args->flags & RKNPU_JOB_PINGPONG ? 1 : 0;
int pc_data_amount_scale = rknpu_dev->config->pc_data_amount_scale;
int pc_task_number_bits = rknpu_dev->config->pc_task_number_bits;
int i = 0;
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
if (i == core_index) {
REG_WRITE((0xe + 0x10000000 * i), 0x1004);
REG_WRITE((0xe + 0x10000000 * i), 0x3004);
}
}
if (!task_obj)
return -EINVAL;
if (job->use_core_num == 1) {
task_start = args->subcore_task[core_index].task_start;
task_end = args->subcore_task[core_index].task_start +
args->subcore_task[core_index].task_end - 1;
task_number = args->subcore_task[core_index].task_number;
} else if (job->use_core_num == 2) {
task_start = args->subcore_task[core_index].task_start;
task_end = args->subcore_task[core_index].task_start +
args->subcore_task[core_index].task_end - 1;
task_number = args->subcore_task[core_index].task_number;
} else if (job->use_core_num == 3) {
task_start = args->subcore_task[core_index + 2].task_start;
task_end = args->subcore_task[core_index + 2].task_start +
args->subcore_task[core_index + 2].task_end - 1;
task_number = args->subcore_task[core_index + 2].task_number;
if (rknpu_dev->config->num_irqs > 1) {
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
if (i == core_index) {
REG_WRITE((0xe + 0x10000000 * i), 0x1004);
REG_WRITE((0xe + 0x10000000 * i), 0x3004);
}
}
if (job->use_core_num == 1) {
task_start = args->subcore_task[core_index].task_start;
task_end = args->subcore_task[core_index].task_start +
args->subcore_task[core_index].task_number -
1;
task_number =
args->subcore_task[core_index].task_number;
} else if (job->use_core_num == 2) {
task_start = args->subcore_task[core_index].task_start;
task_end = args->subcore_task[core_index].task_start +
args->subcore_task[core_index].task_number -
1;
task_number =
args->subcore_task[core_index].task_number;
} else if (job->use_core_num == 3) {
task_start =
args->subcore_task[core_index + 2].task_start;
task_end =
args->subcore_task[core_index + 2].task_start +
args->subcore_task[core_index + 2].task_number -
1;
task_number =
args->subcore_task[core_index + 2].task_number;
}
}
task_base = task_obj->kv_addr;
@@ -217,20 +242,22 @@ static inline int rknpu_job_commit_pc(struct rknpu_job *job, int core_index)
first_task = &task_base[task_start];
last_task = &task_base[task_end];
REG_WRITE(first_task->regcmd_data, RKNPU_OFFSET_PC_DATA_ADDR);
REG_WRITE(first_task->regcmd_addr, RKNPU_OFFSET_PC_DATA_ADDR);
REG_WRITE(first_task->regcfg_amount +
rknpu_dev->config->pc_data_extra_amount - 1,
REG_WRITE((first_task->regcfg_amount + RKNPU_PC_DATA_EXTRA_AMOUNT +
pc_data_amount_scale - 1) /
pc_data_amount_scale -
1,
RKNPU_OFFSET_PC_DATA_AMOUNT);
REG_WRITE(last_task->int_mask, RKNPU_OFFSET_INT_MASK);
REG_WRITE(first_task->int_mask, RKNPU_OFFSET_INT_CLEAR);
REG_WRITE(((0x6 | task_pp_en) << 12) | task_number,
REG_WRITE(((0x6 | task_pp_en) << pc_task_number_bits) | task_number,
RKNPU_OFFSET_PC_TASK_CONTROL);
REG_WRITE(0x0, RKNPU_OFFSET_PC_DMA_BASE_ADDR);
REG_WRITE(args->task_base_addr, RKNPU_OFFSET_PC_DMA_BASE_ADDR);
job->first_task = first_task;
job->last_task = last_task;
@@ -279,7 +306,7 @@ static void rknpu_job_next(struct rknpu_device *rknpu_dev, int core_index)
subcore_data->job = job;
job->run_count--;
job->hw_recoder_time = ktime_get();
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (job->run_count == 0) {
@@ -298,6 +325,7 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
struct rknpu_subcore_data *subcore_data = NULL;
unsigned long flags;
int task_num = 0;
ktime_t now = ktime_get();
subcore_data = &rknpu_dev->subcore_datas[core_index];
task_num = rknn_get_task_number(job, core_index);
@@ -305,6 +333,8 @@ static void rknpu_job_done(struct rknpu_job *job, int ret, int core_index)
subcore_data->job = NULL;
subcore_data->task_num = subcore_data->task_num - task_num;
job->interrupt_count--;
subcore_data->timer.busy_time +=
ktime_us_delta(now, job->hw_recoder_time);
spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags);
if (job->interrupt_count == 0) {
@@ -401,10 +431,12 @@ static void rknpu_job_abort(struct rknpu_job *job)
msleep(100);
if (job->ret == -ETIMEDOUT) {
LOG_ERROR(
"job timeout, irq status: %#x, raw status: %#x, require mask: %#x\n",
"job timeout, irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x\n",
REG_READ(RKNPU_OFFSET_INT_STATUS),
REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
job->int_mask[core_index]);
job->int_mask[core_index],
(REG_READ(RKNPU_OFFSET_PC_TASK_STATUS) &
rknpu_dev->config->pc_task_number_mask));
rknpu_soft_reset(rknpu_dev);
}
for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
@@ -461,18 +493,22 @@ static inline irqreturn_t rknpu_irq_handler(int irq, void *data, int core_index)
return IRQ_HANDLED;
status = REG_READ(RKNPU_OFFSET_INT_STATUS);
REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
job->int_status[core_index] = status;
if (rknpu_fuzz_status(status) != job->int_mask[core_index]) {
LOG_ERROR(
"invalid irq status: %#x, raw status: %#x, require mask: %#x\n",
"invalid irq status: %#x, raw status: %#x, require mask: %#x, task counter: %#x\n",
status, REG_READ(RKNPU_OFFSET_INT_RAW_STATUS),
job->int_mask[core_index]);
job->int_mask[core_index],
(REG_READ(RKNPU_OFFSET_PC_TASK_STATUS) &
rknpu_dev->config->pc_task_number_mask));
REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
return IRQ_HANDLED;
}
REG_WRITE(RKNPU_INT_CLEAR, RKNPU_OFFSET_INT_CLEAR);
rknpu_job_done(job, 0, core_index);
return IRQ_HANDLED;
@@ -541,6 +577,7 @@ static void rknpu_job_timeout_clean(struct rknpu_device *rknpu_dev,
}
}
#ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM
int rknpu_submit_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv)
{
@@ -561,6 +598,7 @@ int rknpu_submit_ioctl(struct drm_device *dev, void *data,
}
if (args->flags & RKNPU_JOB_FENCE_IN) {
#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
struct dma_fence *in_fence;
in_fence = sync_file_get_fence(args->fence_fd);
@@ -589,9 +627,16 @@ int rknpu_submit_ioctl(struct drm_device *dev, void *data,
return ret;
}
#else
LOG_ERROR(
"failed to use rknpu fence, please enable rknpu fence config!\n");
rknpu_job_free(job);
return -EINVAL;
#endif
}
if (args->flags & RKNPU_JOB_FENCE_OUT) {
#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
ret = rknpu_fence_alloc(job);
if (ret) {
rknpu_job_free(job);
@@ -599,6 +644,12 @@ int rknpu_submit_ioctl(struct drm_device *dev, void *data,
}
job->args->fence_fd = rknpu_fence_get_fd(job);
args->fence_fd = job->args->fence_fd;
#else
LOG_ERROR(
"failed to use rknpu fence, please enable rknpu fence config!\n");
rknpu_job_free(job);
return -EINVAL;
#endif
}
if (args->flags & RKNPU_JOB_NONBLOCK) {
@@ -625,6 +676,120 @@ int rknpu_submit_ioctl(struct drm_device *dev, void *data,
return ret;
}
#endif
#ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP
int rknpu_submit_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
{
struct rknpu_submit args;
struct rknpu_job *job = NULL;
int ret = -EINVAL;
if (unlikely(copy_from_user(&args, (struct rknpu_submit *)data,
sizeof(struct rknpu_submit)))) {
LOG_ERROR("%s: copy_from_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
if (args.task_number == 0) {
LOG_ERROR("invalid rknpu task number!\n");
return -EINVAL;
}
job = rknpu_job_alloc(rknpu_dev, &args);
if (!job) {
LOG_ERROR("failed to allocate rknpu job!\n");
return -ENOMEM;
}
if (args.flags & RKNPU_JOB_FENCE_IN) {
#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
struct dma_fence *in_fence;
in_fence = sync_file_get_fence(args.fence_fd);
if (!in_fence) {
LOG_ERROR("invalid fence in fd, fd = %d\n",
args.fence_fd);
return -EINVAL;
}
args.fence_fd = -1;
/*
* Wait if the fence is from a foreign context, or if the fence
* array contains any fence from a foreign context.
*/
ret = 0;
if (!dma_fence_match_context(in_fence,
rknpu_dev->fence_ctx->context))
ret = dma_fence_wait_timeout(in_fence, true,
args.timeout);
dma_fence_put(in_fence);
if (ret < 0) {
if (ret != -ERESTARTSYS)
LOG_ERROR("Error (%d) waiting for fence!\n",
ret);
return ret;
}
#else
LOG_ERROR(
"failed to use rknpu fence, please enable rknpu fence config!\n");
rknpu_job_free(job);
return -EINVAL;
#endif
}
if (args.flags & RKNPU_JOB_FENCE_OUT) {
#ifdef CONFIG_ROCKCHIP_RKNPU_FENCE
ret = rknpu_fence_alloc(job);
if (ret) {
rknpu_job_free(job);
return ret;
}
job->args->fence_fd = rknpu_fence_get_fd(job);
args.fence_fd = job->args->fence_fd;
#else
LOG_ERROR(
"failed to use rknpu fence, please enable rknpu fence config!\n");
rknpu_job_free(job);
return -EINVAL;
#endif
}
if (args.flags & RKNPU_JOB_NONBLOCK) {
job->flags |= RKNPU_JOB_ASYNC;
rknpu_job_timeout_clean(rknpu_dev, job->args->core_mask);
rknpu_job_schedule(job);
ret = job->ret;
if (ret) {
rknpu_job_abort(job);
return ret;
}
} else {
rknpu_job_schedule(job);
if (args.flags & RKNPU_JOB_PC)
job->ret = rknpu_job_wait(job);
args.task_counter = job->args->task_counter;
ret = job->ret;
if (!ret)
rknpu_job_cleanup(job);
else
rknpu_job_abort(job);
}
if (unlikely(copy_to_user((struct rknpu_submit *)data, &args,
sizeof(struct rknpu_submit)))) {
LOG_ERROR("%s: copy_to_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
return ret;
}
#endif
int rknpu_get_hw_version(struct rknpu_device *rknpu_dev, uint32_t *version)
{
@@ -717,6 +882,7 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
uint32_t *dt_rd, uint32_t *wd_rd)
{
void __iomem *rknpu_core_base = rknpu_dev->base[0];
int amount_scale = rknpu_dev->config->pc_data_amount_scale;
if (!rknpu_dev->config->bw_enable) {
LOG_WARN("Get rw_amount is not supported on this device!\n");
@@ -726,13 +892,13 @@ int rknpu_get_rw_amount(struct rknpu_device *rknpu_dev, uint32_t *dt_wr,
spin_lock(&rknpu_dev->lock);
if (dt_wr != NULL)
*dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT);
*dt_wr = REG_READ(RKNPU_OFFSET_DT_WR_AMOUNT) * amount_scale;
if (dt_rd != NULL)
*dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT);
*dt_rd = REG_READ(RKNPU_OFFSET_DT_RD_AMOUNT) * amount_scale;
if (wd_rd != NULL)
*wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT);
*wd_rd = REG_READ(RKNPU_OFFSET_WT_RD_AMOUNT) * amount_scale;
spin_unlock(&rknpu_dev->lock);

205
drivers/rknpu/rknpu_mem.c Normal file
View File

@@ -0,0 +1,205 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
* Author: Felix Zeng <felix.zeng@rock-chips.com>
*/
#include <linux/version.h>
#include <linux/rk-dma-heap.h>
#if KERNEL_VERSION(5, 10, 0) <= LINUX_VERSION_CODE
#include <linux/dma-map-ops.h>
#endif
#include "rknpu_drv.h"
#include "rknpu_ioctl.h"
#include "rknpu_mem.h"
int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
{
struct rknpu_mem_create args;
int ret = -EINVAL;
struct dma_buf_attachment *attachment;
struct sg_table *table;
struct scatterlist *sgl;
dma_addr_t phys;
struct dma_buf *dmabuf;
struct page **pages;
struct page *page;
struct rknpu_mem_object *rknpu_obj = NULL;
int i, fd;
unsigned int length, page_count;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_create *)data,
sizeof(struct rknpu_mem_create)))) {
LOG_ERROR("%s: copy_from_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
if (args.flags & RKNPU_MEM_NON_CONTIGUOUS) {
LOG_ERROR("%s: malloc iommu memory unsupported in current!\n",
__func__);
ret = -EFAULT;
return ret;
}
rknpu_obj = kzalloc(sizeof(*rknpu_obj), GFP_KERNEL);
if (!rknpu_obj)
return PTR_ERR(rknpu_obj);
if (args.handle > 0) {
fd = args.handle;
dmabuf = dma_buf_get(fd);
if (IS_ERR(dmabuf)) {
ret = PTR_ERR(dmabuf);
goto err_free_obj;
}
rknpu_obj->dmabuf = dmabuf;
rknpu_obj->owner = 0;
} else {
/* Start test kernel alloc/free dma buf */
dmabuf = rk_dma_heap_buffer_alloc(rknpu_dev->heap, args.size,
O_CLOEXEC | O_RDWR, 0x0,
dev_name(rknpu_dev->dev));
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
rknpu_obj->dmabuf = dmabuf;
rknpu_obj->owner = 1;
fd = dma_buf_fd(dmabuf, O_CLOEXEC | O_RDWR);
if (fd < 0) {
ret = -EFAULT;
goto err_free_dma_buf;
}
}
attachment = dma_buf_attach(dmabuf, rknpu_dev->dev);
if (IS_ERR(attachment)) {
ret = PTR_ERR(attachment);
goto err_free_dma_buf;
}
table = dma_buf_map_attachment(attachment, DMA_BIDIRECTIONAL);
if (IS_ERR(table)) {
ret = PTR_ERR(table);
goto err_free_dma_buf;
}
for_each_sgtable_sg(table, sgl, i) {
phys = sg_dma_address(sgl);
page = sg_page(sgl);
LOG_DEBUG("%s, %d, phys = %pad, length = 0x%x\n", __func__,
__LINE__, &phys, sg_dma_len(sgl));
length = sg_dma_len(sgl);
}
page_count = length >> PAGE_SHIFT;
pages = kmalloc_array(page_count, sizeof(struct page), GFP_KERNEL);
for (i = 0; i < page_count; i++)
pages[i] = &page[i];
rknpu_obj->kv_addr = vmap(pages, page_count, VM_MAP, PAGE_KERNEL);
kfree(pages);
dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
dma_buf_detach(dmabuf, attachment);
rknpu_obj->size = PAGE_ALIGN(args.size);
rknpu_obj->dma_addr = phys;
rknpu_obj->sgt = table;
args.size = rknpu_obj->size;
args.obj_addr = (__u64)(uintptr_t)rknpu_obj;
args.dma_addr = rknpu_obj->dma_addr;
args.handle = fd;
LOG_DEBUG(
"args.handle = %d, args.size = %lld, rknpu_obj = %#llx, rknpu_obj->dma_addr = %#llx\n",
args.handle, args.size, (__u64)(uintptr_t)rknpu_obj,
(__u64)rknpu_obj->dma_addr);
if (unlikely(copy_to_user((struct rknpu_mem_create *)data, &args,
sizeof(struct rknpu_mem_create)))) {
LOG_ERROR("%s: copy_to_user failed\n", __func__);
ret = -EFAULT;
goto err_free_dma_buf;
}
return 0;
err_free_dma_buf:
dma_buf_put(dmabuf);
if (rknpu_obj->owner)
rk_dma_heap_buffer_free(dmabuf);
return ret;
err_free_obj:
kfree(rknpu_obj);
return ret;
}
int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
{
struct rknpu_mem_object *rknpu_obj = NULL;
struct rknpu_mem_destroy args;
struct dma_buf *dmabuf;
int ret = -EFAULT;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_destroy *)data,
sizeof(struct rknpu_mem_destroy)))) {
LOG_ERROR("%s: copy_from_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
if (!kern_addr_valid(args.obj_addr)) {
LOG_ERROR("%s: invalid params, unknown obj_addr\n", __func__);
ret = -EFAULT;
return ret;
}
rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr;
dmabuf = rknpu_obj->dmabuf;
LOG_DEBUG(
"free args.handle = %d, rknpu_obj = %#llx, rknpu_obj->dma_addr = %#llx\n",
args.handle, (__u64)(uintptr_t)rknpu_obj,
(__u64)rknpu_obj->dma_addr);
vunmap(rknpu_obj->kv_addr);
kfree(rknpu_obj);
return 0;
}
int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data)
{
struct rknpu_mem_object *rknpu_obj = NULL;
struct rknpu_mem_sync args;
struct dma_buf *dmabuf;
int ret = -EFAULT;
if (unlikely(copy_from_user(&args, (struct rknpu_mem_sync *)data,
sizeof(struct rknpu_mem_sync)))) {
LOG_ERROR("%s: copy_from_user failed\n", __func__);
ret = -EFAULT;
return ret;
}
rknpu_obj = (struct rknpu_mem_object *)(uintptr_t)args.obj_addr;
dmabuf = rknpu_obj->dmabuf;
if (args.flags & RKNPU_MEM_SYNC_TO_DEVICE) {
dmabuf->ops->end_cpu_access_partial(dmabuf, DMA_TO_DEVICE,
args.offset, args.size);
}
if (args.flags & RKNPU_MEM_SYNC_FROM_DEVICE) {
dmabuf->ops->begin_cpu_access_partial(dmabuf, DMA_FROM_DEVICE,
args.offset, args.size);
}
return 0;
}