From c125c678f6a4da7ad0e794da538be2eb7e82e467 Mon Sep 17 00:00:00 2001 From: Yandong Lin Date: Fri, 21 Jan 2022 14:32:08 +0800 Subject: [PATCH] video: rockchip: mpp: support av1 decode Signed-off-by: Yandong Lin Signed-off-by: Simon Xue Change-Id: I76b54488c9078688ebc9c4df902e6940c95f3594 --- drivers/video/rockchip/mpp/Kconfig | 5 + drivers/video/rockchip/mpp/Makefile | 1 + drivers/video/rockchip/mpp/mpp_av1dec.c | 1300 +++++++++++++++++++ drivers/video/rockchip/mpp/mpp_common.c | 1 + drivers/video/rockchip/mpp/mpp_common.h | 10 +- drivers/video/rockchip/mpp/mpp_iommu.c | 2 +- drivers/video/rockchip/mpp/mpp_iommu.h | 1 + drivers/video/rockchip/mpp/mpp_iommu_av1d.c | 944 ++++++++++++++ drivers/video/rockchip/mpp/mpp_service.c | 7 +- 9 files changed, 2268 insertions(+), 3 deletions(-) create mode 100644 drivers/video/rockchip/mpp/mpp_av1dec.c create mode 100644 drivers/video/rockchip/mpp/mpp_iommu_av1d.c diff --git a/drivers/video/rockchip/mpp/Kconfig b/drivers/video/rockchip/mpp/Kconfig index e5d6f57f716c..f46894b9cbf3 100644 --- a/drivers/video/rockchip/mpp/Kconfig +++ b/drivers/video/rockchip/mpp/Kconfig @@ -65,4 +65,9 @@ config ROCKCHIP_MPP_JPGDEC help rockchip mpp rkv jpeg decoder. +config ROCKCHIP_MPP_AV1DEC + bool "AV1 decoder device driver" + help + rockchip mpp av1 decoder. + endif diff --git a/drivers/video/rockchip/mpp/Makefile b/drivers/video/rockchip/mpp/Makefile index bd9e0a0d8952..16e191a8e976 100644 --- a/drivers/video/rockchip/mpp/Makefile +++ b/drivers/video/rockchip/mpp/Makefile @@ -21,6 +21,7 @@ rk_vcodec-$(CONFIG_ROCKCHIP_MPP_VDPU2) += mpp_vdpu2.o rk_vcodec-$(CONFIG_ROCKCHIP_MPP_VEPU2) += mpp_vepu2.o rk_vcodec-$(CONFIG_ROCKCHIP_MPP_IEP2) += mpp_iep2.o rk_vcodec-$(CONFIG_ROCKCHIP_MPP_JPGDEC) += mpp_jpgdec.o +rk_vcodec-$(CONFIG_ROCKCHIP_MPP_AV1DEC) += mpp_av1dec.o mpp_iommu_av1d.o # hack for workaround rk_vcodec-$(CONFIG_CPU_PX30) += hack/mpp_hack_px30.o diff --git a/drivers/video/rockchip/mpp/mpp_av1dec.c b/drivers/video/rockchip/mpp/mpp_av1dec.c new file mode 100644 index 000000000000..4ea41aa1ea7f --- /dev/null +++ b/drivers/video/rockchip/mpp/mpp_av1dec.c @@ -0,0 +1,1300 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2021 Fuzhou Rockchip Electronics Co., Ltd + * + * author: + * Ding Wei, leo.ding@rock-chips.com + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mpp_debug.h" +#include "mpp_common.h" +#include "mpp_iommu.h" + +#define AV1DEC_DRIVER_NAME "mpp_av1dec" + +#define AV1DEC_SESSION_MAX_BUFFERS 40 + +/* REG_DEC_INT, bits for interrupt */ +#define AV1DEC_INT_PIC_INF BIT(24) +#define AV1DEC_INT_TIMEOUT BIT(18) +#define AV1DEC_INT_SLICE BIT(17) +#define AV1DEC_INT_STRM_ERROR BIT(16) +#define AV1DEC_INT_ASO_ERROR BIT(15) +#define AV1DEC_INT_BUF_EMPTY BIT(14) +#define AV1DEC_INT_BUS_ERROR BIT(13) +#define AV1DEC_DEC_INT BIT(12) +#define AV1DEC_DEC_INT_RAW BIT(8) +#define AV1DEC_DEC_IRQ_DIS BIT(4) +#define AV1DEC_DEC_START BIT(0) + +#define MPP_ALIGN(x, a) (((x)+(a)-1)&~((a)-1)) +/* REG_DEC_EN, bit for gate */ +#define AV1DEC_CLOCK_GATE_EN BIT(10) + +#define to_av1dec_info(info) \ + container_of(info, struct av1dec_hw_info, hw) +#define to_av1dec_task(ctx) \ + container_of(ctx, struct av1dec_task, mpp_task) +#define to_av1dec_dev(dev) \ + container_of(dev, struct av1dec_dev, mpp) + +/* define functions */ +#define MPP_GET_BITS(v, p, b) (((v) >> (p)) & ((1 << (b)) - 1)) +#define MPP_BASE_TO_IDX(a) ((a) / sizeof(u32)) + +enum AV1DEC_CLASS_TYPE { + AV1DEC_CLASS_VCD = 0, + AV1DEC_CLASS_CACHE = 1, + AV1DEC_CLASS_AFBC = 2, + AV1DEC_CLASS_BUTT, +}; + +enum av1dec_trans_type { + AV1DEC_TRANS_BASE = 0x0000, + + AV1DEC_TRANS_VCD = AV1DEC_TRANS_BASE + 0, + AV1DEC_TRANS_CACHE = AV1DEC_TRANS_BASE + 1, + AV1DEC_TRANS_AFBC = AV1DEC_TRANS_BASE + 2, + AV1DEC_TRANS_BUTT, +}; + +struct av1dec_hw_info { + struct mpp_hw_info hw; + /* register range by class */ + u32 reg_class_num; + struct { + u32 base_s; + u32 base_e; + } reg_class[AV1DEC_CLASS_BUTT]; + /* fd translate for class */ + u32 trans_class_num; + struct { + u32 class; + u32 trans_fmt; + } trans_class[AV1DEC_TRANS_BUTT]; + + /* interrupt config register */ + int int_base; + /* enable hardware register */ + int en_base; + /* status register */ + int sta_base; + /* clear irq register */ + int clr_base; + /* stream register */ + int strm_base; + + u32 err_mask; +}; + +struct av1dec_task { + struct mpp_task mpp_task; + + struct av1dec_hw_info *hw_info; + /* for malloc register data buffer */ + u32 *reg_data; + /* class register */ + struct { + u32 valid; + u32 base; + u32 *data; + /* offset base reg_data */ + u32 off; + /* length for class */ + u32 len; + } reg_class[AV1DEC_CLASS_BUTT]; + /* register offset info */ + struct reg_offset_info off_inf; + + enum MPP_CLOCK_MODE clk_mode; + u32 irq_status; + /* req for current task */ + u32 w_req_cnt; + struct mpp_request w_reqs[MPP_MAX_MSG_NUM]; + u32 r_req_cnt; + struct mpp_request r_reqs[MPP_MAX_MSG_NUM]; +}; + +struct av1dec_dev { + struct mpp_dev mpp; + struct av1dec_hw_info *hw_info; + + struct mpp_clk_info aclk_info; + struct mpp_clk_info hclk_info; + u32 default_max_load; +#ifdef CONFIG_PROC_FS + struct proc_dir_entry *procfs; +#endif + struct reset_control *rst_a; + struct reset_control *rst_h; + + void __iomem *reg_base[AV1DEC_CLASS_BUTT]; + int irq[AV1DEC_CLASS_BUTT]; +}; + +static struct av1dec_hw_info av1dec_hw_info = { + .hw = { + .reg_num = 512, + .reg_id = 0, + .reg_en = 1, + .reg_start = 1, + .reg_end = 319, + }, + .reg_class_num = 3, + .reg_class[AV1DEC_CLASS_VCD] = { + .base_s = 0x0000, + .base_e = 0x07fc, + }, + .reg_class[AV1DEC_CLASS_CACHE] = { + .base_s = 0x10000, + .base_e = 0x10294, + }, + .reg_class[AV1DEC_CLASS_AFBC] = { + .base_s = 0x20000, + .base_e = 0x2034c, + }, + .trans_class_num = AV1DEC_TRANS_BUTT, + .trans_class[AV1DEC_CLASS_VCD] = { + .class = AV1DEC_CLASS_VCD, + .trans_fmt = AV1DEC_TRANS_VCD, + }, + .trans_class[AV1DEC_CLASS_CACHE] = { + .class = AV1DEC_CLASS_CACHE, + .trans_fmt = AV1DEC_TRANS_CACHE, + }, + .trans_class[AV1DEC_CLASS_AFBC] = { + .class = AV1DEC_CLASS_AFBC, + .trans_fmt = AV1DEC_TRANS_AFBC, + }, + .int_base = 0x0004, + .en_base = 0x0004, + .sta_base = 0x0004, + .clr_base = 0x0004, + .strm_base = 0x02a4, + .err_mask = 0x7e000, +}; + +/* + * file handle translate information for v2 + */ +static const u16 trans_tbl_av1_vcd[] = { + 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, + 93, 95, 97, 99, 101, 103, 105, 107, 109, 111, 113, 115, + 117, 133, 135, 137, 139, 141, 143, 145, 147, + 167, 169, 171, 173, 175, 177, 179, 183, 190, 192, 194, + 196, 198, 200, 202, 204, 224, 226, 228, 230, 232, 234, + 236, 238, 326, 328, 339, 341, 348, 350, 505, 507 +}; + +static const u16 trans_tbl_av1_cache[] = { + 13, 18, 23, 28, 33, 38, 43, 48, 53, 58, 63, 68, 73, 78, 83, 88, + 134, 135, 138, 139, 142, 143, 146, 147, +}; + +static const u16 trans_tbl_av1_afbc[] = { + 32, 33, 34, 35, 48, 49, 50, 51, 96, 97, 98, 99 +}; + +static struct mpp_trans_info trans_av1dec[] = { + [AV1DEC_TRANS_VCD] = { + .count = ARRAY_SIZE(trans_tbl_av1_vcd), + .table = trans_tbl_av1_vcd, + }, + [AV1DEC_TRANS_CACHE] = { + .count = ARRAY_SIZE(trans_tbl_av1_cache), + .table = trans_tbl_av1_cache, + }, + [AV1DEC_TRANS_AFBC] = { + .count = ARRAY_SIZE(trans_tbl_av1_afbc), + .table = trans_tbl_av1_afbc, + }, +}; + +static bool req_over_class(struct mpp_request *req, + struct av1dec_task *task, int class) +{ + bool ret; + u32 base_s, base_e, req_e; + struct av1dec_hw_info *hw = task->hw_info; + + if (class > hw->reg_class_num) + return false; + + base_s = hw->reg_class[class].base_s; + base_e = hw->reg_class[class].base_e; + req_e = req->offset + req->size - sizeof(u32); + + ret = (req->offset <= base_e && req_e >= base_s) ? true : false; + + return ret; +} + +static int av1dec_alloc_reg_class(struct av1dec_task *task) +{ + int i; + u32 data_size; + struct av1dec_hw_info *hw = task->hw_info; + + data_size = 0; + for (i = 0; i < hw->reg_class_num; i++) { + u32 base_s = hw->reg_class[i].base_s; + u32 base_e = hw->reg_class[i].base_e; + + task->reg_class[i].base = base_s; + task->reg_class[i].off = data_size; + task->reg_class[i].len = base_e - base_s + sizeof(u32); + data_size += task->reg_class[i].len; + } + + task->reg_data = kzalloc(data_size, GFP_KERNEL); + if (!task->reg_data) + return -ENOMEM; + + for (i = 0; i < hw->reg_class_num; i++) + task->reg_class[i].data = task->reg_data + (task->reg_class[i].off / sizeof(u32)); + + return 0; +} + +static int av1dec_update_req(struct av1dec_task *task, int class, + struct mpp_request *req_in, + struct mpp_request *req_out) +{ + u32 base_s, base_e, req_e, s, e; + struct av1dec_hw_info *hw = task->hw_info; + + if (class > hw->reg_class_num) + return -EINVAL; + + base_s = hw->reg_class[class].base_s; + base_e = hw->reg_class[class].base_e; + req_e = req_in->offset + req_in->size - sizeof(u32); + s = max(req_in->offset, base_s); + e = min(req_e, base_e); + + req_out->offset = s; + req_out->size = e - s + sizeof(u32); + req_out->data = (u8 *)req_in->data + (s - req_in->offset); + mpp_debug(DEBUG_TASK_INFO, "req_out->offset=%08x, req_out->size=%d\n", + req_out->offset, req_out->size); + + return 0; +} + +static int av1dec_extract_task_msg(struct av1dec_task *task, + struct mpp_task_msgs *msgs) +{ + int ret; + u32 i; + struct mpp_request *req; + struct av1dec_hw_info *hw = task->hw_info; + + mpp_debug_enter(); + + mpp_debug(DEBUG_TASK_INFO, "req_cnt=%d, set_cnt=%d, poll_cnt=%d, reg_class=%d\n", + msgs->req_cnt, msgs->set_cnt, msgs->poll_cnt, hw->reg_class_num); + + for (i = 0; i < msgs->req_cnt; i++) { + req = &msgs->reqs[i]; + mpp_debug(DEBUG_TASK_INFO, "msg: cmd %08x, offset %08x, size %d\n", + req->cmd, req->offset, req->size); + if (!req->size) + continue; + + switch (req->cmd) { + case MPP_CMD_SET_REG_WRITE: { + u32 class; + u32 base, *regs; + struct mpp_request *wreq; + + for (class = 0; class < hw->reg_class_num; class++) { + if (!req_over_class(req, task, class)) + continue; + mpp_debug(DEBUG_TASK_INFO, "found write_calss %d\n", class); + wreq = &task->w_reqs[task->w_req_cnt]; + av1dec_update_req(task, class, req, wreq); + + base = task->reg_class[class].base; + regs = (u32 *)task->reg_class[class].data; + regs += MPP_BASE_TO_IDX(req->offset - base); + if (copy_from_user(regs, wreq->data, wreq->size)) { + mpp_err("copy_from_user fail, offset %08x\n", wreq->offset); + ret = -EIO; + goto fail; + } + task->w_req_cnt++; + } + } break; + case MPP_CMD_SET_REG_READ: { + u32 class; + struct mpp_request *rreq; + + for (class = 0; class < hw->reg_class_num; class++) { + if (!req_over_class(req, task, class)) + continue; + mpp_debug(DEBUG_TASK_INFO, "found read_calss %d\n", class); + rreq = &task->r_reqs[task->r_req_cnt]; + av1dec_update_req(task, class, req, rreq); + task->r_req_cnt++; + } + } break; + case MPP_CMD_SET_REG_ADDR_OFFSET: { + mpp_extract_reg_offset_info(&task->off_inf, req); + } break; + default: + break; + } + } + mpp_debug(DEBUG_TASK_INFO, "w_req_cnt=%d, r_req_cnt=%d\n", + task->w_req_cnt, task->r_req_cnt); + + mpp_debug_leave(); + return 0; + +fail: + mpp_debug_leave(); + return ret; +} + +static void *av1dec_alloc_task(struct mpp_session *session, + struct mpp_task_msgs *msgs) +{ + int ret; + u32 i, j; + struct mpp_task *mpp_task = NULL; + struct av1dec_task *task = NULL; + struct mpp_dev *mpp = session->mpp; + + mpp_debug_enter(); + + task = kzalloc(sizeof(*task), GFP_KERNEL); + if (!task) + return NULL; + + mpp_task = &task->mpp_task; + mpp_task_init(session, mpp_task); + mpp_task->hw_info = mpp->var->hw_info; + task->hw_info = to_av1dec_info(mpp_task->hw_info); + + /* alloc reg data for task */ + ret = av1dec_alloc_reg_class(task); + if (ret) + goto free_task; + mpp_task->reg = task->reg_class[0].data; + /* extract reqs for current task */ + ret = av1dec_extract_task_msg(task, msgs); + if (ret) + goto free_reg_class; + + /* process fd in register */ + if (!(msgs->flags & MPP_FLAGS_REG_FD_NO_TRANS)) { + int cnt; + const u16 *tbl; + u32 offset; + struct av1dec_hw_info *hw = task->hw_info; + + for (i = 0; i < task->w_req_cnt; i++) { + struct mpp_request *req = &task->w_reqs[i]; + + for (i = 0; i < hw->trans_class_num; i++) { + u32 class = hw->trans_class[i].class; + u32 fmt = hw->trans_class[i].trans_fmt; + u32 *reg = task->reg_class[class].data; + u32 base_idx = MPP_BASE_TO_IDX(task->reg_class[class].base); + + if (!req_over_class(req, task, i)) + continue; + mpp_debug(DEBUG_TASK_INFO, "class=%d, base_idx=%d\n", + class, base_idx); + if (!reg) + continue; + + ret = mpp_translate_reg_address(session, mpp_task, fmt, reg, NULL); + if (ret) + goto fail; + + cnt = mpp->var->trans_info[fmt].count; + tbl = mpp->var->trans_info[fmt].table; + for (j = 0; j < cnt; j++) { + offset = mpp_query_reg_offset_info(&task->off_inf, + tbl[j] + base_idx); + mpp_debug(DEBUG_IOMMU, + "reg[%d] + offset %d\n", tbl[j] + base_idx, offset); + reg[tbl[j]] += offset; + } + } + } + } + task->clk_mode = CLK_MODE_NORMAL; + + mpp_debug_leave(); + + return mpp_task; + +fail: + mpp_task_dump_mem_region(mpp, mpp_task); + mpp_task_dump_reg(mpp, mpp_task); + mpp_task_finalize(session, mpp_task); +free_reg_class: + kfree(task->reg_data); +free_task: + kfree(task); + + return NULL; +} +#define AV1_PP_CONFIG_INDEX 321 +#define AV1_PP_TILE_SIZE GENMASK_ULL(10, 9) +#define AV1_PP_TILE_16X16 BIT(10) + +#define AV1_PP_OUT_LUMA_ADR_INDEX 326 +#define AV1_PP_OUT_CHROMA_ADR_INDEX 328 + +#define AV1_L2_CACHE_SHAPER_CTRL 0x20 +#define AV1_L2_CACHE_SHAPER_EN BIT(0) +#define AV1_L2_CACHE_PP0_Y_CONFIG0 0x84 +#define AV1_L2_CACHE_PP0_Y_CONFIG2 0x8c +#define AV1_L2_CACHE_PP0_Y_CONFIG3 0x90 +#define AV1_L2_CACHE_PP0_U_CONFIG0 0x98 +#define AV1_L2_CACHE_PP0_U_CONFIG2 0xa0 +#define AV1_L2_CACHE_PP0_U_CONFIG3 0xa4 + +#define AV1_L2_CACHE_RD_ONLY_CTRL 0x204 +#define AV1_L2_CACHE_RD_ONLY_CONFIG 0x208 + +static int av1dec_set_l2_cache(struct av1dec_dev *dec, struct av1dec_task *task) +{ + int val; + u32 *regs = (u32 *)task->reg_class[0].data; + u32 width = (regs[4] >> 19) * 8; + u32 height = ((regs[4] >> 6) & 0x1fff) * 8; + u32 pixel_width = (((regs[322]) >> 27) & 0x1F) == 1 ? 8 : 16; + u32 pre_fetch_height = 136; + u32 max_h; + u32 line_cnt; + u32 line_size; + u32 line_stride; + + /* channel 4, PPU0_Y Configuration */ + /* afbc sharper can't use open cache. + * afbc out must be tile 16x16. + */ + if ((regs[AV1_PP_CONFIG_INDEX] & AV1_PP_TILE_SIZE) != AV1_PP_TILE_16X16) { + line_size = MPP_ALIGN(MPP_ALIGN(width * pixel_width, 8) / 8, 16); + line_stride = MPP_ALIGN(MPP_ALIGN(width * pixel_width, 8) / 8, 16) >> 4; + line_cnt = height; + max_h = pre_fetch_height; + + writel_relaxed(regs[AV1_PP_OUT_LUMA_ADR_INDEX] + 0x1, + dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_PP0_Y_CONFIG0); + val = line_size | (line_stride << 16); + writel_relaxed(val, dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_PP0_Y_CONFIG2); + + val = line_cnt | (max_h << 16); + writel_relaxed(val, dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_PP0_Y_CONFIG3); + + /* channel 5, PPU0_U Configuration */ + line_size = MPP_ALIGN(MPP_ALIGN(width * pixel_width, 8) / 8, 16); + line_stride = MPP_ALIGN(MPP_ALIGN(width * pixel_width, 8) / 8, 16) >> 4; + line_cnt = height >> 1; + max_h = pre_fetch_height >> 1; + + writel_relaxed(regs[AV1_PP_OUT_CHROMA_ADR_INDEX] + 0x1, + dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_PP0_U_CONFIG0); + val = line_size | (line_stride << 16); + writel_relaxed(val, dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_PP0_U_CONFIG2); + + val = line_cnt | (max_h << 16); + writel_relaxed(val, dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_PP0_U_CONFIG3); + /* shaper enable */ + writel_relaxed(AV1_L2_CACHE_SHAPER_EN, + dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_SHAPER_CTRL); + } + + /* TODO: set exception list */ + + /* multi id enable bit */ + writel_relaxed(0x00000001, dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_RD_ONLY_CONFIG); + /* reorder_e and cache_e */ + writel_relaxed(0x00000081, dec->reg_base[AV1DEC_CLASS_CACHE] + AV1_L2_CACHE_RD_ONLY_CTRL); + /* wmb */ + wmb(); + + return 0; +} +#define REG_CONTROL 0x20 +#define REG_INTRENBL 0x34 +#define REG_ACKNOWLEDGE 0x38 +#define REG_FORMAT 0x100 +#define REG_COMPRESSENABLE 0x340 +#define REG_HEADERBASE 0x80 +#define REG_PAYLOADBASE 0xC0 +#define REG_INPUTBUFBASE 0x180 +#define REG_INPUTBUFSTRIDE 0x200 +#define REG_INPUTBUFSIZE 0x140 + +static int av1dec_set_afbc(struct av1dec_dev *dec, struct av1dec_task *task) +{ + u32 *regs = (u32 *)task->reg_class[0].data; + u32 width = (regs[4] >> 19) * 8; + u32 height = ((regs[4] >> 6) & 0x1fff) * 8; + u32 pixel_width_y, pixel_width_c, pixel_width = 8; + u32 vir_top = (((regs[503]) >> 16) & 0xf); + u32 vir_left = (((regs[503]) >> 20) & 0xf); + u32 vir_bottom = (((regs[503]) >> 24) & 0xf); + u32 vir_right = (((regs[503]) >> 28) & 0xf); + u32 fbc_format = 0; + u32 fbc_stream_number = 0; + u32 fbc_comp_en[2] = {0, 0}; + u32 pp_width_final[2] = {0, 0}; + u32 pp_height_final[2] = {0, 0}; + u32 pp_hdr_base[2] = {0, 0}; + u32 pp_payload_base[2] = {0, 0}; + u32 pp_input_base[2] = {0, 0}; + u32 pp_input_stride[2] = {0, 0}; + u32 bus_address; + u32 i = 0; + + pixel_width_y = ((regs[8] >> 6) & 0x3) + 8; + pixel_width_c = ((regs[8] >> 4) & 0x3) + 8; + pixel_width = (pixel_width_y == 8 && pixel_width_c == 8) ? 8 : 10; + + if ((regs[AV1_PP_CONFIG_INDEX] & AV1_PP_TILE_SIZE) == AV1_PP_TILE_16X16) { + u32 offset = MPP_ALIGN((vir_left + width + vir_right) * + (height + 28) / 16, 64); + + bus_address = regs[505]; + fbc_stream_number++; + if (pixel_width == 10) + fbc_format = 3; + else + fbc_format = 9; + fbc_comp_en[0] = 1; + fbc_comp_en[1] = 1; + + pp_width_final[0] = pp_width_final[1] = vir_left + width + vir_right; + pp_height_final[0] = pp_height_final[1] = vir_top + height + vir_bottom; + + if (pixel_width == 10) + pp_input_stride[0] = pp_input_stride[1] = 2 * pp_width_final[0]; + else + pp_input_stride[0] = pp_input_stride[1] = pp_width_final[0]; + + pp_hdr_base[0] = pp_hdr_base[1] = bus_address; + pp_payload_base[0] = pp_payload_base[1] = bus_address + offset; + pp_input_base[0] = pp_input_base[1] = bus_address; + + writel_relaxed((fbc_stream_number << 9), + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_CONTROL); + writel_relaxed(0x1, dec->reg_base[AV1DEC_CLASS_AFBC] + REG_INTRENBL); + + for (i = 0; i < 2; i++) { + writel_relaxed(fbc_format, + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_FORMAT + i * 4); + writel_relaxed(fbc_comp_en[i], dec->reg_base[AV1DEC_CLASS_AFBC] + + REG_COMPRESSENABLE + i * 4); + /* hdr base */ + writel_relaxed(pp_hdr_base[i], + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_HEADERBASE + i * 4); + /* payload */ + writel_relaxed(pp_payload_base[i], + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_PAYLOADBASE + i * 4); + /* bufsize */ + writel_relaxed(((pp_height_final[i] << 15) | pp_width_final[i]), + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_INPUTBUFSIZE + i * 4); + /* buf */ + writel_relaxed(pp_input_base[i], + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_INPUTBUFBASE + i * 4); + /* stride */ + writel_relaxed(pp_input_stride[i], dec->reg_base[AV1DEC_CLASS_AFBC] + + REG_INPUTBUFSTRIDE + i * 4); + } + /* wmb */ + wmb(); + writel(((fbc_stream_number << 9) | (1 << 7)), + dec->reg_base[AV1DEC_CLASS_AFBC] + REG_CONTROL); /* update */ + writel((fbc_stream_number << 9), dec->reg_base[AV1DEC_CLASS_AFBC] + REG_CONTROL); + + } + return 0; +} + +static int av1dec_run(struct mpp_dev *mpp, struct mpp_task *mpp_task) +{ + int i; + u32 en_val = 0; + struct av1dec_dev *dec = to_av1dec_dev(mpp); + struct av1dec_hw_info *hw = dec->hw_info; + struct av1dec_task *task = to_av1dec_task(mpp_task); + + mpp_debug_enter(); + mpp_iommu_flush_tlb(mpp->iommu_info); + av1dec_set_l2_cache(dec, task); + av1dec_set_afbc(dec, task); + + for (i = 0; i < task->w_req_cnt; i++) { + int class; + struct mpp_request *req = &task->w_reqs[i]; + + for (class = 0; class < hw->reg_class_num; class++) { + int j, s, e; + u32 base, *regs; + + if (!req_over_class(req, task, class)) + continue; + base = task->reg_class[class].base; + s = MPP_BASE_TO_IDX(req->offset - base); + e = s + req->size / sizeof(u32); + regs = (u32 *)task->reg_class[class].data; + + mpp_debug(DEBUG_TASK_INFO, "found rd_class %d, base=%08x, s=%d, e=%d\n", + class, base, s, e); + for (j = s; j < e; j++) { + if (class == 0 && j == hw->hw.reg_en) { + en_val = regs[j]; + continue; + } + writel_relaxed(regs[j], dec->reg_base[class] + j * sizeof(u32)); + } + } + } + + /* init current task */ + mpp->cur_task = mpp_task; + /* Flush the register before the start the device */ + wmb(); + mpp_write(mpp, hw->en_base, en_val); + + mpp_debug_leave(); + + return 0; +} + +static int av1dec_vcd_irq(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + struct av1dec_hw_info *hw = dec->hw_info; + + mpp_debug_enter(); + + mpp->irq_status = mpp_read(mpp, hw->sta_base); + if (!mpp->irq_status) + return IRQ_NONE; + + mpp_write(mpp, hw->clr_base, 0); + + mpp_debug_leave(); + + return IRQ_WAKE_THREAD; +} + +static int av1dec_isr(struct mpp_dev *mpp) +{ + struct mpp_task *mpp_task = mpp->cur_task; + struct av1dec_dev *dec = to_av1dec_dev(mpp); + struct av1dec_task *task = to_av1dec_task(mpp_task); + u32 *regs = (u32 *)task->reg_class[0].data; + + mpp_debug_enter(); + + /* FIXME use a spin lock here */ + if (!mpp_task) { + dev_err(mpp->dev, "no current task\n"); + return IRQ_HANDLED; + } + + mpp_time_diff(mpp_task); + mpp->cur_task = NULL; + + /* clear l2 cache status */ + writel_relaxed(0x0, dec->reg_base[AV1DEC_CLASS_CACHE] + 0x020); + writel_relaxed(0x0, dec->reg_base[AV1DEC_CLASS_CACHE] + 0x204); + /* multi id enable bit */ + writel_relaxed(0x00000000, dec->reg_base[AV1DEC_CLASS_CACHE] + 0x208); + + if (((regs[321] >> 9) & 0x3) == 0x2) { + u32 ack_status = readl(dec->reg_base[AV1DEC_CLASS_AFBC] + REG_ACKNOWLEDGE); + + if ((ack_status & 0x1) == 0x1) { + u32 ctl_val = readl(dec->reg_base[AV1DEC_CLASS_AFBC] + REG_CONTROL); + + ctl_val |= 1; + writel_relaxed(ctl_val, dec->reg_base[AV1DEC_CLASS_AFBC] + REG_CONTROL); + } + } + task->irq_status = mpp->irq_status; + mpp_debug(DEBUG_IRQ_STATUS, "irq_status: %08x\n", task->irq_status); + if (task->irq_status & dec->hw_info->err_mask) { + atomic_inc(&mpp->reset_request); + /* dump register */ + if (mpp_debug_unlikely(DEBUG_DUMP_ERR_REG)) { + mpp_debug(DEBUG_DUMP_ERR_REG, "irq_status: %08x\n", + task->irq_status); + mpp_task_dump_hw_reg(mpp); + } + } + mpp_task_finish(mpp_task->session, mpp_task); + + mpp_debug_leave(); + + return IRQ_HANDLED; +} + +static int av1dec_finish(struct mpp_dev *mpp, + struct mpp_task *mpp_task) +{ + u32 i; + struct av1dec_task *task = to_av1dec_task(mpp_task); + struct av1dec_dev *dec = to_av1dec_dev(mpp); + struct av1dec_hw_info *hw = dec->hw_info; + + mpp_debug_enter(); + + for (i = 0; i < task->r_req_cnt; i++) { + int class; + struct mpp_request *req = &task->r_reqs[i]; + + for (class = 0; class < hw->reg_class_num; class++) { + int j, s, e; + u32 base, *regs; + + if (!req_over_class(req, task, class)) + continue; + base = task->reg_class[class].base; + s = MPP_BASE_TO_IDX(req->offset - base); + e = s + req->size / sizeof(u32); + regs = (u32 *)task->reg_class[class].data; + + mpp_debug(DEBUG_TASK_INFO, "found rd_class %d, base=%08x, s=%d, e=%d\n", + class, base, s, e); + for (j = s; j < e; j++) { + /* revert hack for irq status */ + if (class == 0 && j == MPP_BASE_TO_IDX(hw->sta_base)) { + regs[j] = task->irq_status; + continue; + } + regs[j] = readl_relaxed(dec->reg_base[class] + j * sizeof(u32)); + } + } + } + + mpp_debug_leave(); + + return 0; +} + +static int av1dec_result(struct mpp_dev *mpp, + struct mpp_task *mpp_task, + struct mpp_task_msgs *msgs) +{ + u32 i; + struct av1dec_task *task = to_av1dec_task(mpp_task); + struct av1dec_dev *dec = to_av1dec_dev(mpp); + struct av1dec_hw_info *hw = dec->hw_info; + + mpp_debug_enter(); + + for (i = 0; i < task->r_req_cnt; i++) { + int class; + struct mpp_request *req = &task->r_reqs[i]; + + for (class = 0; class < hw->reg_class_num; class++) { + u32 base, *regs; + + if (!req_over_class(req, task, class)) + continue; + base = task->reg_class[class].base; + regs = (u32 *)task->reg_class[class].data; + regs += MPP_BASE_TO_IDX(req->offset - base); + + if (copy_to_user(req->data, regs, req->size)) { + mpp_err("copy_to_user reg fail\n"); + return -EIO; + } + } + } + mpp_debug_leave(); + + return 0; +} + +static int av1dec_free_task(struct mpp_session *session, + struct mpp_task *mpp_task) +{ + struct av1dec_task *task = to_av1dec_task(mpp_task); + + mpp_task_finalize(session, mpp_task); + kfree(task->reg_data); + kfree(task); + + return 0; +} + +#ifdef CONFIG_PROC_FS +static int av1dec_procfs_remove(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + + if (dec->procfs) { + proc_remove(dec->procfs); + dec->procfs = NULL; + } + + return 0; +} + +static int av1dec_procfs_init(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + + dec->procfs = proc_mkdir(mpp->dev->of_node->name, mpp->srv->procfs); + if (IS_ERR_OR_NULL(dec->procfs)) { + mpp_err("failed on open procfs\n"); + dec->procfs = NULL; + return -EIO; + } + /* for debug */ + mpp_procfs_create_u32("aclk", 0644, + dec->procfs, &dec->aclk_info.debug_rate_hz); + mpp_procfs_create_u32("session_buffers", 0644, + dec->procfs, &mpp->session_max_buffers); + + return 0; +} +#else +static inline int av1dec_procfs_remove(struct mpp_dev *mpp) +{ + return 0; +} + +static inline int av1dec_procfs_init(struct mpp_dev *mpp) +{ + return 0; +} +#endif + +static int av1dec_init(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + int ret = 0; + + /* Get clock info from dtsi */ + ret = mpp_get_clk_info(mpp, &dec->aclk_info, "aclk_vcodec"); + if (ret) + mpp_err("failed on clk_get aclk_vcodec\n"); + ret = mpp_get_clk_info(mpp, &dec->hclk_info, "hclk_vcodec"); + if (ret) + mpp_err("failed on clk_get hclk_vcodec\n"); + + /* Get normal max workload from dtsi */ + of_property_read_u32(mpp->dev->of_node, + "rockchip,default-max-load", + &dec->default_max_load); + /* Set default rates */ + mpp_set_clk_info_rate_hz(&dec->aclk_info, CLK_MODE_DEFAULT, 300 * MHZ); + + /* Get reset control from dtsi */ + dec->rst_a = mpp_reset_control_get(mpp, RST_TYPE_A, "video_a"); + if (!dec->rst_a) + mpp_err("No aclk reset resource define\n"); + dec->rst_h = mpp_reset_control_get(mpp, RST_TYPE_H, "video_h"); + if (!dec->rst_h) + mpp_err("No hclk reset resource define\n"); + + return 0; +} + +static int av1dec_reset(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + + mpp_debug_enter(); + + if (dec->rst_a && dec->rst_h) { + rockchip_pmu_idle_request(mpp->dev, true); + mpp_safe_reset(dec->rst_a); + mpp_safe_reset(dec->rst_h); + udelay(5); + mpp_safe_unreset(dec->rst_a); + mpp_safe_unreset(dec->rst_h); + rockchip_pmu_idle_request(mpp->dev, false); + } + + mpp_debug_leave(); + + return 0; +} + +static int av1dec_clk_on(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + + mpp_clk_safe_enable(dec->aclk_info.clk); + mpp_clk_safe_enable(dec->hclk_info.clk); + + return 0; +} + +static int av1dec_clk_off(struct mpp_dev *mpp) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + + clk_disable_unprepare(dec->aclk_info.clk); + clk_disable_unprepare(dec->hclk_info.clk); + + return 0; +} + +static int av1dec_set_freq(struct mpp_dev *mpp, + struct mpp_task *mpp_task) +{ + struct av1dec_dev *dec = to_av1dec_dev(mpp); + struct av1dec_task *task = to_av1dec_task(mpp_task); + + mpp_clk_set_rate(&dec->aclk_info, task->clk_mode); + + return 0; +} + +static struct mpp_hw_ops av1dec_hw_ops = { + .init = av1dec_init, + .clk_on = av1dec_clk_on, + .clk_off = av1dec_clk_off, + .set_freq = av1dec_set_freq, + .reset = av1dec_reset, +}; + +static struct mpp_dev_ops av1dec_dev_ops = { + .alloc_task = av1dec_alloc_task, + .run = av1dec_run, + .irq = av1dec_vcd_irq, + .isr = av1dec_isr, + .finish = av1dec_finish, + .result = av1dec_result, + .free_task = av1dec_free_task, +}; +static const struct mpp_dev_var av1dec_data = { + .device_type = MPP_DEVICE_AV1DEC, + .hw_info = &av1dec_hw_info.hw, + .trans_info = trans_av1dec, + .hw_ops = &av1dec_hw_ops, + .dev_ops = &av1dec_dev_ops, +}; + +static const struct of_device_id mpp_av1dec_dt_match[] = { + { + .compatible = "rockchip,av1-decoder", + .data = &av1dec_data, + }, + {}, +}; + +static int av1dec_device_match(struct device *dev, struct device_driver *drv) +{ + return 1; +} + +static int av1dec_device_probe(struct device *dev) +{ + const struct platform_driver *drv; + struct platform_device *pdev = to_platform_device(dev); + + drv = to_platform_driver(dev->driver); + + drv->probe(pdev); + + return 0; +} + +static int av1dec_device_remove(struct device *dev) +{ + + struct platform_device *pdev = to_platform_device(dev); + struct platform_driver *drv = to_platform_driver(dev->driver); + + if (dev->driver && drv->remove) + drv->remove(pdev); + return 0; +} + +static void av1dec_device_shutdown(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct platform_driver *drv = to_platform_driver(dev->driver); + + if (dev->driver && drv->shutdown) + drv->shutdown(pdev); +} + +static int av1dec_dma_configure(struct device *dev) +{ + return of_dma_configure(dev, dev->of_node, true); +} + +static const struct dev_pm_ops platform_dev_pm_ops = { + .runtime_suspend = pm_generic_runtime_suspend, + .runtime_resume = pm_generic_runtime_resume, +}; + +struct bus_type av1dec_bus = { + .name = "av1dec_bus", + .match = av1dec_device_match, + .probe = av1dec_device_probe, + .remove = av1dec_device_remove, + .shutdown = av1dec_device_shutdown, + .dma_configure = av1dec_dma_configure, + .pm = &platform_dev_pm_ops, +}; + +static int av1_of_device_add(struct platform_device *ofdev) +{ + WARN_ON(ofdev->dev.of_node == NULL); + + /* name and id have to be set so that the platform bus doesn't get + * confused on matching + */ + ofdev->name = dev_name(&ofdev->dev); + ofdev->id = PLATFORM_DEVID_NONE; + + /* + * If this device has not binding numa node in devicetree, that is + * of_node_to_nid returns NUMA_NO_NODE. device_add will assume that this + * device is on the same node as the parent. + */ + set_dev_node(&ofdev->dev, of_node_to_nid(ofdev->dev.of_node)); + + return device_add(&ofdev->dev); +} + +struct platform_device *av1dec_device_create(void) +{ + int ret = -ENODEV; + struct device_node *root, *child; + struct platform_device *pdev; + + root = of_find_node_by_path("/"); + + for_each_child_of_node(root, child) { + if (!of_match_node(mpp_av1dec_dt_match, child)) + continue; + + pr_info("Adding child %pOF\n", child); + + pdev = of_device_alloc(child, "av1d-master", NULL); + if (!pdev) + return ERR_PTR(-ENOMEM); + + pdev->dev.bus = &av1dec_bus; + + dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + + ret = av1_of_device_add(pdev); + if (ret) { + platform_device_put(pdev); + return ERR_PTR(-EINVAL); + } + + pr_info("register device %s\n", dev_name(&pdev->dev)); + + return pdev; + } + + return ERR_PTR(ret); +} + +int av1dec_driver_register(struct platform_driver *drv) +{ + int ret; + /* 1. register bus */ + ret = bus_register(&av1dec_bus); + if (ret) { + pr_err("failed to register av1 bus: %d\n", ret); + return ret; + } + /* 2. register iommu driver */ + platform_driver_register(&rockchip_av1_iommu_driver); + /* 3. create device */ + av1dec_device_create(); + /* 4. register av1 driver */ + return driver_register(&drv->driver); +} + +static irqreturn_t av1dec_cache_irq(int irq, void *dev_id) +{ + struct av1dec_dev *dec = dev_id; + u32 shaper_st, rd_st; + + shaper_st = readl(dec->reg_base[AV1DEC_CLASS_CACHE] + 0x2c); + rd_st = readl(dec->reg_base[AV1DEC_CLASS_CACHE] + 0x204); + + mpp_debug(DEBUG_IRQ_STATUS, "cache irq st shaper 0x%x read 0x%x\n", shaper_st, rd_st); + + writel(shaper_st, dec->reg_base[AV1DEC_CLASS_CACHE] + 0x2c); + writel(rd_st, dec->reg_base[AV1DEC_CLASS_CACHE] + 0x204); + + return IRQ_HANDLED; +} + +static int av1dec_cache_init(struct platform_device *pdev, struct av1dec_dev *dec) +{ + int ret; + struct resource *res; + struct device *dev = &pdev->dev; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cache"); + if (!res) + return -ENOMEM; + + dec->reg_base[AV1DEC_CLASS_CACHE] = devm_ioremap(dev, res->start, resource_size(res)); + if (!dec->reg_base[AV1DEC_CLASS_CACHE]) { + dev_err(dev, "ioremap failed for resource %pR\n", res); + return -EINVAL; + } + + dec->irq[AV1DEC_CLASS_CACHE] = platform_get_irq(pdev, 1); + + ret = devm_request_irq(dev, dec->irq[AV1DEC_CLASS_CACHE], + av1dec_cache_irq, IRQF_SHARED, "irq_cache", dec); + if (ret) + mpp_err("ret=%d\n", ret); + return ret; +} + +static int av1dec_afbc_init(struct platform_device *pdev, struct av1dec_dev *dec) +{ + struct resource *res; + struct device *dev = &pdev->dev; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "afbc"); + if (!res) + return -ENOMEM; + + dec->reg_base[AV1DEC_CLASS_AFBC] = devm_ioremap(dev, res->start, resource_size(res)); + if (!dec->reg_base[AV1DEC_CLASS_AFBC]) { + dev_err(dev, "ioremap failed for resource %pR\n", res); + return -EINVAL; + } + dec->irq[AV1DEC_CLASS_AFBC] = platform_get_irq(pdev, 2); + + return 0; +} + +static int av1dec_probe(struct platform_device *pdev) +{ + int ret = 0; + struct device *dev = &pdev->dev; + struct av1dec_dev *dec = NULL; + struct mpp_dev *mpp = NULL; + const struct of_device_id *match = NULL; + + dev_info(dev, "probing start\n"); + + dec = devm_kzalloc(dev, sizeof(*dec), GFP_KERNEL); + if (!dec) + return -ENOMEM; + + mpp = &dec->mpp; + platform_set_drvdata(pdev, dec); + + if (pdev->dev.of_node) { + match = of_match_node(mpp_av1dec_dt_match, pdev->dev.of_node); + if (match) + mpp->var = (struct mpp_dev_var *)match->data; + } + /* get vcd resource */ + ret = mpp_dev_probe(mpp, pdev); + if (ret) + return ret; + mpp->iommu_info->skip_refresh = 1; + dec->reg_base[AV1DEC_CLASS_VCD] = mpp->reg_base; + ret = devm_request_threaded_irq(dev, mpp->irq, + mpp_dev_irq, + mpp_dev_isr_sched, + IRQF_SHARED, + dev_name(dev), mpp); + if (ret) { + dev_err(dev, "register interrupter runtime failed\n"); + goto failed_get_irq; + } + dec->irq[AV1DEC_CLASS_VCD] = mpp->irq; + /* get cache resource */ + ret = av1dec_cache_init(pdev, dec); + if (ret) + goto failed_get_irq; + /* get afbc resource */ + ret = av1dec_afbc_init(pdev, dec); + if (ret) + goto failed_get_irq; + mpp->session_max_buffers = AV1DEC_SESSION_MAX_BUFFERS; + dec->hw_info = to_av1dec_info(mpp->var->hw_info); + av1dec_procfs_init(mpp); + mpp_dev_register_srv(mpp, mpp->srv); + dev_info(dev, "probing finish\n"); + pm_runtime_get(mpp->dev); + + return 0; + +failed_get_irq: + mpp_dev_remove(mpp); + + return ret; +} + +static int av1dec_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct av1dec_dev *dec = platform_get_drvdata(pdev); + + dev_info(dev, "remove device\n"); + mpp_dev_remove(&dec->mpp); + av1dec_procfs_remove(&dec->mpp); + pm_runtime_put(dec->mpp.dev); + + return 0; +} + +static void av1dec_shutdown(struct platform_device *pdev) +{ + int ret; + int val; + struct device *dev = &pdev->dev; + struct av1dec_dev *dec = platform_get_drvdata(pdev); + struct mpp_dev *mpp = &dec->mpp; + + dev_info(dev, "shutdown device\n"); + + atomic_inc(&mpp->srv->shutdown_request); + ret = readx_poll_timeout(atomic_read, + &mpp->task_count, + val, val == 0, 1000, 200000); + if (ret == -ETIMEDOUT) + dev_err(dev, "wait total running time out\n"); + + dev_info(dev, "shutdown success\n"); +} + +struct platform_driver rockchip_av1dec_driver = { + .probe = av1dec_probe, + .remove = av1dec_remove, + .shutdown = av1dec_shutdown, + .driver = { + .name = AV1DEC_DRIVER_NAME, + .of_match_table = of_match_ptr(mpp_av1dec_dt_match), + .bus = &av1dec_bus, + }, +}; diff --git a/drivers/video/rockchip/mpp/mpp_common.c b/drivers/video/rockchip/mpp/mpp_common.c index a27919f499fe..fc5816a84573 100644 --- a/drivers/video/rockchip/mpp/mpp_common.c +++ b/drivers/video/rockchip/mpp/mpp_common.c @@ -68,6 +68,7 @@ const char *mpp_device_name[MPP_DEVICE_BUTT] = { [MPP_DEVICE_VDPU2] = "VDPU2", [MPP_DEVICE_VDPU1_PP] = "VDPU1_PP", [MPP_DEVICE_VDPU2_PP] = "VDPU2_PP", + [MPP_DEVICE_AV1DEC] = "AV1DEC", [MPP_DEVICE_HEVC_DEC] = "HEVC_DEC", [MPP_DEVICE_RKVDEC] = "RKVDEC", [MPP_DEVICE_AVSPLUS_DEC] = "AVSPLUS_DEC", diff --git a/drivers/video/rockchip/mpp/mpp_common.h b/drivers/video/rockchip/mpp/mpp_common.h index 1027ff1f944d..09ffe954ba4a 100644 --- a/drivers/video/rockchip/mpp/mpp_common.h +++ b/drivers/video/rockchip/mpp/mpp_common.h @@ -51,7 +51,8 @@ enum MPP_DEVICE_TYPE { MPP_DEVICE_VDPU1 = 0, /* 0x00000001 */ MPP_DEVICE_VDPU2 = 1, /* 0x00000002 */ MPP_DEVICE_VDPU1_PP = 2, /* 0x00000004 */ - MPP_DEVICE_VDPU2_PP = 3, /* 0x00000008 */ + MPP_DEVICE_VDPU2_PP = 3, /* 0x00000008 */ + MPP_DEVICE_AV1DEC = 4, /* 0x00000010 */ MPP_DEVICE_HEVC_DEC = 8, /* 0x00000100 */ MPP_DEVICE_RKVDEC = 9, /* 0x00000200 */ @@ -84,6 +85,7 @@ enum MPP_DRIVER_TYPE { MPP_DRIVER_JPGDEC, MPP_DRIVER_RKVDEC2, MPP_DRIVER_RKVENC2, + MPP_DRIVER_AV1DEC, MPP_DRIVER_BUTT, }; @@ -826,5 +828,11 @@ extern struct platform_driver rockchip_iep2_driver; extern struct platform_driver rockchip_jpgdec_driver; extern struct platform_driver rockchip_rkvdec2_driver; extern struct platform_driver rockchip_rkvenc2_driver; +extern struct platform_driver rockchip_av1dec_driver; +extern struct platform_driver rockchip_av1_iommu_driver; + +extern struct platform_device *av1dec_device_create(void); +extern int av1dec_driver_register(struct platform_driver *drv); +extern struct bus_type av1dec_bus; #endif diff --git a/drivers/video/rockchip/mpp/mpp_iommu.c b/drivers/video/rockchip/mpp/mpp_iommu.c index 9fd8196123ac..0be561ed1a9f 100644 --- a/drivers/video/rockchip/mpp/mpp_iommu.c +++ b/drivers/video/rockchip/mpp/mpp_iommu.c @@ -468,7 +468,7 @@ int mpp_iommu_refresh(struct mpp_iommu_info *info, struct device *dev) { int ret; - if (!info) + if (!info || info->skip_refresh) return 0; /* disable iommu */ diff --git a/drivers/video/rockchip/mpp/mpp_iommu.h b/drivers/video/rockchip/mpp/mpp_iommu.h index 9ae1b6ba3567..34ff97a92ec2 100644 --- a/drivers/video/rockchip/mpp/mpp_iommu.h +++ b/drivers/video/rockchip/mpp/mpp_iommu.h @@ -73,6 +73,7 @@ struct mpp_iommu_info { struct iommu_group *group; struct mpp_rk_iommu *iommu; iommu_fault_handler_t hdl; + u32 skip_refresh; }; struct mpp_dma_session * diff --git a/drivers/video/rockchip/mpp/mpp_iommu_av1d.c b/drivers/video/rockchip/mpp/mpp_iommu_av1d.c new file mode 100644 index 000000000000..0e10b3966efb --- /dev/null +++ b/drivers/video/rockchip/mpp/mpp_iommu_av1d.c @@ -0,0 +1,944 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Compatible with the IOMMU of av1 decode + * + * Module Authors: Yandong Lin + * Simon Xue + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mpp_debug.h" +#include "mpp_common.h" + +struct av1_iommu_domain { + struct list_head iommus; + u32 *dt; /* page directory table */ + dma_addr_t dt_dma; + spinlock_t iommus_lock; /* lock for iommus list */ + spinlock_t dt_lock; /* lock for modifying page directory table */ + struct iommu_domain domain; + /* for av1 iommu */ + u64 *pta; /* page directory table */ + dma_addr_t pta_dma; +}; + +struct av1_iommu { + struct device *dev; + void __iomem **bases; + int num_mmu; + int num_irq; + struct clk_bulk_data *clocks; + int num_clocks; + struct iommu_device iommu; + struct list_head node; /* entry in rk_iommu_domain.iommus */ + struct iommu_domain *domain; /* domain to which iommu is attached */ + struct iommu_group *group; +}; + +struct av1_iommudata { + struct device_link *link; /* runtime PM link from IOMMU to master */ + struct av1_iommu *iommu; + bool defer_attach; +}; + +#define RK_IOMMU_AV1 0xa +#define NUM_DT_ENTRIES 1024 +#define NUM_PT_ENTRIES 1024 + +#define SPAGE_ORDER 12 +#define SPAGE_SIZE (1 << SPAGE_ORDER) + +/* av1 iommu regs address */ +#define AV1_CLOCK_CTRL_BASE 0x0 +#define AV1_IDLE_ST_BASE 0x4 +#define AV1_MMU_CONFIG0_BASE 0x184 +#define AV1_MMU_CONFIG1_BASE 0x1ac +#define AV1_MMU_AHB_EXCEPTION_BASE 0x380 +#define AV1_MMU_AHB_STATUS_BASE 0x384 +#define AV1_MMU_AHB_CONTROL_BASE 0x388 +#define AV1_MMU_AHB_TBL_ARRAY_BASE_L_BASE 0x38C +#define AV1_MMU_AHB_TBL_ARRAY_BASE_H_BASE 0x390 +#define AV1_MMU_AHB_CTX_PD_BASE 0x3b4 +#define AV1_MMU_BUTT_BASE 0xffff + +/* MMU register offsets */ +#define AV1_MMU_FLUSH_BASE 0x184 +#define AV1_MMU_BIT_FLUSH BIT(4) + +#define AV1_MMU_PAGE_FAULT_ADDR 0x380 +#define AV1_MMU_STATUS_BASE 0x384 /* IRQ status */ + +#define AV1_MMU_EN_BASE 0x388 +#define AV1_MMU_BIT_ENABLE BIT(0) + +#define AV1_MMU_OUT_OF_BOUND BIT(28) +/* Irq mask */ +#define AV1_MMU_IRQ_MASK 0x7 + +#define AV1_DTE_PT_ADDRESS_MASK 0xffffffc0 +#define AV1_DTE_PT_VALID BIT(0) + +#define AV1_PAGE_DESC_LO_MASK 0xfffff000 +#define AV1_PAGE_DESC_HI_MASK GENMASK_ULL(39, 32) +#define AV1_PAGE_DESC_HI_SHIFT (32-4) + +#define AV1_IOMMU_PGSIZE_BITMAP 0x007ff000 + +static inline phys_addr_t av1_dte_pt_address(u32 dte) +{ + return (phys_addr_t)dte & AV1_DTE_PT_ADDRESS_MASK; +} + +static inline u32 av1_mk_dte(dma_addr_t pt_dma) +{ + return (pt_dma) | AV1_DTE_PT_VALID; +} + +#define AV1_PTE_PAGE_ADDRESS_MASK 0xfffffff0 +#define AV1_PTE_PAGE_WRITABLE BIT(2) +#define AV1_PTE_PAGE_VALID BIT(0) + +static struct device *dma_dev; + +static inline phys_addr_t av1_pte_page_address(u32 pte) +{ + u64 pte_av1 = pte; + + pte_av1 = ((pte_av1 & AV1_PAGE_DESC_HI_MASK) << AV1_PAGE_DESC_HI_SHIFT) | + (pte_av1 & AV1_PAGE_DESC_LO_MASK); + + return (phys_addr_t)pte_av1; +} + +static u32 av1_mk_pte(phys_addr_t page, int prot) +{ + u32 flags = 0; + + flags |= (prot & IOMMU_WRITE) ? AV1_PTE_PAGE_WRITABLE : 0; + page = (page & AV1_PAGE_DESC_LO_MASK) | + ((page & AV1_PAGE_DESC_HI_MASK) >> AV1_PAGE_DESC_HI_SHIFT); + page &= AV1_PTE_PAGE_ADDRESS_MASK; + + return page | flags | AV1_PTE_PAGE_VALID; +} + +#define AV1_DTE_PT_VALID BIT(0) + +static inline bool av1_dte_is_pt_valid(u32 dte) +{ + return dte & AV1_DTE_PT_VALID; +} + +static inline bool av1_pte_is_page_valid(u32 pte) +{ + return pte & AV1_PTE_PAGE_VALID; +} + +static u32 av1_mk_pte_invalid(u32 pte) +{ + return pte & ~AV1_PTE_PAGE_VALID; +} + +#define AV1_MASTER_TLB_MASK GENMASK_ULL(31, 10) +/* mode 0 : 4k */ +#define AV1_PTA_4K_MODE 0 + +static struct av1_iommu *av1_iommu_from_dev(struct device *dev) +{ + struct av1_iommudata *data = dev_iommu_priv_get(dev); + + return data ? data->iommu : NULL; +} + +static u64 av1_mk_pta(dma_addr_t dt_dma) +{ + u64 val = (dt_dma & AV1_MASTER_TLB_MASK) | AV1_PTA_4K_MODE; + + return val; +} + +static struct av1_iommu_domain *to_av1_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct av1_iommu_domain, domain); +} + +static void av1_iommu_disable(struct av1_iommu *iommu) +{ + int i; + + /* Ignore error while disabling, just keep going */ + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); + for (i = 0; i < iommu->num_mmu; i++) + writel(0, iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE); + + clk_bulk_disable(iommu->num_clocks, iommu->clocks); +} + +static int av1_iommu_enable(struct av1_iommu *iommu) +{ + struct iommu_domain *domain = iommu->domain; + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + int ret, i; + + ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks); + if (ret) + return ret; + + for (i = 0; i < iommu->num_mmu; i++) { + u32 val = readl(iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE); + + if (!(val & AV1_MMU_BIT_ENABLE)) { + writel(av1_domain->pta_dma, + iommu->bases[i] + AV1_MMU_AHB_TBL_ARRAY_BASE_L_BASE); + writel(AV1_MMU_OUT_OF_BOUND, iommu->bases[i] + AV1_MMU_CONFIG1_BASE); + writel(AV1_MMU_BIT_ENABLE, iommu->bases[i] + AV1_MMU_AHB_EXCEPTION_BASE); + writel(AV1_MMU_BIT_ENABLE, iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE); + } + } + clk_bulk_disable(iommu->num_clocks, iommu->clocks); + return ret; +} + +static inline void av1_table_flush(struct av1_iommu_domain *dom, dma_addr_t dma, + unsigned int count) +{ + size_t size = count * sizeof(u32); /* count of u32 entry */ + + dma_sync_single_for_device(dma_dev, dma, size, DMA_TO_DEVICE); +} + +#define AV1_IOVA_DTE_MASK 0xffc00000 +#define AV1_IOVA_DTE_SHIFT 22 +#define AV1_IOVA_PTE_MASK 0x003ff000 +#define AV1_IOVA_PTE_SHIFT 12 +#define AV1_IOVA_PAGE_MASK 0x00000fff +#define AV1_IOVA_PAGE_SHIFT 0 + +static u32 av1_iova_dte_index(dma_addr_t iova) +{ + return (u32)(iova & AV1_IOVA_DTE_MASK) >> AV1_IOVA_DTE_SHIFT; +} + +static u32 av1_iova_pte_index(dma_addr_t iova) +{ + return (u32)(iova & AV1_IOVA_PTE_MASK) >> AV1_IOVA_PTE_SHIFT; +} + +static u32 av1_iova_page_offset(dma_addr_t iova) +{ + return (u32)(iova & AV1_IOVA_PAGE_MASK) >> AV1_IOVA_PAGE_SHIFT; +} + +static u32 av1_iommu_read(void __iomem *base, u32 offset) +{ + return readl(base + offset); +} + +static void av1_iommu_write(void __iomem *base, u32 offset, u32 value) +{ + writel(value, base + offset); +} + + +static void av1_iommu_flush_tlb_all(struct iommu_domain *domain) +{ + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + struct list_head *pos; + unsigned long flags; + int i; + + spin_lock_irqsave(&av1_domain->iommus_lock, flags); + list_for_each(pos, &av1_domain->iommus) { + struct av1_iommu *iommu; + int ret; + + iommu = list_entry(pos, struct av1_iommu, node); + ret = pm_runtime_get_if_in_use(iommu->dev); + if (WARN_ON_ONCE(ret < 0)) + continue; + if (ret) { + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); + for (i = 0; i < iommu->num_mmu; i++) { + writel(AV1_MMU_BIT_FLUSH, + iommu->bases[i] + AV1_MMU_FLUSH_BASE); + writel(0, iommu->bases[i] + AV1_MMU_FLUSH_BASE); + } + clk_bulk_disable(iommu->num_clocks, iommu->clocks); + pm_runtime_put(iommu->dev); + } + } + spin_unlock_irqrestore(&av1_domain->iommus_lock, flags); +} + +static irqreturn_t av1_iommu_irq(int irq, void *dev_id) +{ + struct av1_iommu *iommu = dev_id; + u32 int_status; + dma_addr_t iova; + irqreturn_t ret = IRQ_NONE; + int i, err; + + err = pm_runtime_get_if_in_use(iommu->dev); + if (!err || WARN_ON_ONCE(err < 0)) + return ret; + + if (WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks))) + goto out; + + for (i = 0; i < iommu->num_mmu; i++) { + int_status = av1_iommu_read(iommu->bases[i], AV1_MMU_STATUS_BASE); + if (int_status & AV1_MMU_IRQ_MASK) { + dev_err(iommu->dev, "unexpected int_status=%08x\n", int_status); + iova = av1_iommu_read(iommu->bases[i], AV1_MMU_PAGE_FAULT_ADDR); + + if (iommu->domain) + report_iommu_fault(iommu->domain, iommu->dev, iova, int_status); + else + dev_err(iommu->dev, + "Page fault while iommu not attached to domain?\n"); + } + av1_iommu_write(iommu->bases[i], AV1_MMU_STATUS_BASE, 0); + ret = IRQ_HANDLED; + } + + clk_bulk_disable(iommu->num_clocks, iommu->clocks); + +out: + pm_runtime_put(iommu->dev); + return ret; +} + +static bool av1_iommu_is_attach_deferred(struct iommu_domain *domain, + struct device *dev) +{ + struct av1_iommudata *data = dev_iommu_priv_get(dev); + + return data->defer_attach; +} + +static struct iommu_domain *av1_iommu_domain_alloc(unsigned type) +{ + struct av1_iommu_domain *av1_domain; + + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) + return NULL; + + if (!dma_dev) + return NULL; + + av1_domain = kzalloc(sizeof(*av1_domain), GFP_KERNEL); + if (!av1_domain) + return NULL; + + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&av1_domain->domain)) + goto err_free_domain; + + /* + * av132xx iommus use a 2 level pagetable. + * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. + * Allocate one 4 KiB page for each table. + */ + av1_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); + if (!av1_domain->dt) + goto err_put_cookie; + + av1_domain->dt_dma = dma_map_single(dma_dev, av1_domain->dt, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, av1_domain->dt_dma)) { + dev_err(dma_dev, "DMA map error for DT\n"); + goto err_free_dt; + } + + av1_domain->pta = (u64 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); + if (!av1_domain->pta) + goto err_unmap_dt; + + av1_domain->pta_dma = dma_map_single(dma_dev, av1_domain->pta, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, av1_domain->pta_dma)) { + dev_err(dma_dev, "DMA map error for PTA\n"); + goto err_free_pta; + } + av1_domain->pta[0] = av1_mk_pta(av1_domain->dt_dma); + + av1_table_flush(av1_domain, av1_domain->pta_dma, 1024); + av1_table_flush(av1_domain, av1_domain->dt_dma, NUM_DT_ENTRIES); + + spin_lock_init(&av1_domain->iommus_lock); + spin_lock_init(&av1_domain->dt_lock); + INIT_LIST_HEAD(&av1_domain->iommus); + + av1_domain->domain.geometry.aperture_start = 0; + av1_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); + av1_domain->domain.geometry.force_aperture = true; + + return &av1_domain->domain; +err_free_pta: + free_page((unsigned long)av1_domain->pta); +err_unmap_dt: + dma_unmap_single(dma_dev, av1_domain->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); +err_free_dt: + free_page((unsigned long)av1_domain->dt); +err_put_cookie: + if (type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&av1_domain->domain); +err_free_domain: + kfree(av1_domain); + + return NULL; +} + +static phys_addr_t av1_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + unsigned long flags; + phys_addr_t pt_phys, phys = 0; + u32 dte, pte; + u32 *page_table; + + spin_lock_irqsave(&av1_domain->dt_lock, flags); + + dte = av1_domain->dt[av1_iova_dte_index(iova)]; + if (!av1_dte_is_pt_valid(dte)) + goto out; + + pt_phys = av1_dte_pt_address(dte); + page_table = (u32 *)phys_to_virt(pt_phys); + pte = page_table[av1_iova_pte_index(iova)]; + if (!av1_pte_is_page_valid(pte)) + goto out; + + phys = av1_pte_page_address(pte) + av1_iova_page_offset(iova); +out: + spin_unlock_irqrestore(&av1_domain->dt_lock, flags); + + return phys; +} + +static u32 *av1_dte_get_page_table(struct av1_iommu_domain *av1_domain, dma_addr_t iova) +{ + u32 *page_table, *dte_addr; + u32 dte_index, dte; + phys_addr_t pt_phys; + dma_addr_t pt_dma; + + assert_spin_locked(&av1_domain->dt_lock); + + dte_index = av1_iova_dte_index(iova); + dte_addr = &av1_domain->dt[dte_index]; + dte = *dte_addr; + if (av1_dte_is_pt_valid(dte)) + goto done; + + page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); + if (!page_table) + return ERR_PTR(-ENOMEM); + + pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, pt_dma)) { + dev_err(dma_dev, "DMA mapping error while allocating page table\n"); + free_page((unsigned long)page_table); + return ERR_PTR(-ENOMEM); + } + + dte = av1_mk_dte(pt_dma); + *dte_addr = dte; + + av1_table_flush(av1_domain, pt_dma, NUM_PT_ENTRIES); + av1_table_flush(av1_domain, + av1_domain->dt_dma + dte_index * sizeof(u32), 1); +done: + pt_phys = av1_dte_pt_address(dte); + return (u32 *)phys_to_virt(pt_phys); +} + +static size_t av1_iommu_unmap_iova(struct av1_iommu_domain *av1_domain, + u32 *pte_addr, dma_addr_t pte_dma, + size_t size) +{ + unsigned int pte_count; + unsigned int pte_total = size / SPAGE_SIZE; + + assert_spin_locked(&av1_domain->dt_lock); + + for (pte_count = 0; pte_count < pte_total; pte_count++) { + u32 pte = pte_addr[pte_count]; + + if (!av1_pte_is_page_valid(pte)) + break; + + pte_addr[pte_count] = av1_mk_pte_invalid(pte); + } + + av1_table_flush(av1_domain, pte_dma, pte_count); + + return pte_count * SPAGE_SIZE; +} + +static int av1_iommu_map_iova(struct av1_iommu_domain *av1_domain, u32 *pte_addr, + dma_addr_t pte_dma, dma_addr_t iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned int pte_count; + unsigned int pte_total = size / SPAGE_SIZE; + phys_addr_t page_phys; + + assert_spin_locked(&av1_domain->dt_lock); + + for (pte_count = 0; pte_count < pte_total; pte_count++) { + u32 pte = pte_addr[pte_count]; + + if (av1_pte_is_page_valid(pte)) + goto unwind; + + pte_addr[pte_count] = av1_mk_pte(paddr, prot); + + paddr += SPAGE_SIZE; + } + + av1_table_flush(av1_domain, pte_dma, pte_total); + + return 0; +unwind: + /* Unmap the range of iovas that we just mapped */ + av1_iommu_unmap_iova(av1_domain, pte_addr, pte_dma, + pte_count * SPAGE_SIZE); + + iova += pte_count * SPAGE_SIZE; + page_phys = av1_pte_page_address(pte_addr[pte_count]); + pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", + &iova, &page_phys, &paddr, prot); + + return -EADDRINUSE; +} + +static size_t av1_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, + size_t size, struct iommu_iotlb_gather *gather) +{ + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + unsigned long flags; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; + phys_addr_t pt_phys; + u32 dte; + u32 *pte_addr; + size_t unmap_size; + + spin_lock_irqsave(&av1_domain->dt_lock, flags); + + dte = av1_domain->dt[av1_iova_dte_index(iova)]; + /* Just return 0 if iova is unmapped */ + if (!av1_dte_is_pt_valid(dte)) { + spin_unlock_irqrestore(&av1_domain->dt_lock, flags); + return 0; + } + + pt_phys = av1_dte_pt_address(dte); + pte_addr = (u32 *)phys_to_virt(pt_phys) + av1_iova_pte_index(iova); + pte_dma = pt_phys + av1_iova_pte_index(iova) * sizeof(u32); + unmap_size = av1_iommu_unmap_iova(av1_domain, pte_addr, pte_dma, size); + + spin_unlock_irqrestore(&av1_domain->dt_lock, flags); + + return unmap_size; +} + +static int av1_iommu_map(struct iommu_domain *domain, unsigned long _iova, + phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +{ + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + unsigned long flags; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; + u32 *page_table, *pte_addr; + u32 dte, pte_index; + int ret; + + spin_lock_irqsave(&av1_domain->dt_lock, flags); + + page_table = av1_dte_get_page_table(av1_domain, iova); + if (IS_ERR(page_table)) { + spin_unlock_irqrestore(&av1_domain->dt_lock, flags); + return PTR_ERR(page_table); + } + + dte = av1_domain->dt[av1_iova_dte_index(iova)]; + pte_index = av1_iova_pte_index(iova); + pte_addr = &page_table[pte_index]; + pte_dma = av1_dte_pt_address(dte) + pte_index * sizeof(u32); + ret = av1_iommu_map_iova(av1_domain, pte_addr, pte_dma, iova, + paddr, size, prot); + + spin_unlock_irqrestore(&av1_domain->dt_lock, flags); + + return ret; +} + +static void av1_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct av1_iommu *iommu; + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + unsigned long flags; + int ret; + + /* Allow 'virtual devices' (eg drm) to detach from domain */ + iommu = av1_iommu_from_dev(dev); + if (WARN_ON(!iommu)) + return; + + dev_dbg(dev, "Detaching from iommu domain\n"); + + if (!iommu->domain) + return; + + spin_lock_irqsave(&av1_domain->iommus_lock, flags); + list_del_init(&iommu->node); + spin_unlock_irqrestore(&av1_domain->iommus_lock, flags); + + ret = pm_runtime_get_if_in_use(iommu->dev); + WARN_ON_ONCE(ret < 0); + if (ret > 0) { + av1_iommu_disable(iommu); + pm_runtime_put(iommu->dev); + } + iommu->domain = NULL; +} + +static int av1_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct av1_iommu *iommu; + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + unsigned long flags; + int ret; + + iommu = av1_iommu_from_dev(dev); + if (WARN_ON(!iommu)) + return -ENODEV; + + if (iommu->domain) + av1_iommu_detach_device(iommu->domain, dev); + + iommu->domain = domain; + + /* Attach NULL for disable iommu */ + if (!domain) + return 0; + + spin_lock_irqsave(&av1_domain->iommus_lock, flags); + list_add_tail(&iommu->node, &av1_domain->iommus); + spin_unlock_irqrestore(&av1_domain->iommus_lock, flags); + + ret = pm_runtime_get_if_in_use(iommu->dev); + if (!ret || WARN_ON_ONCE(ret < 0)) + return 0; + + ret = av1_iommu_enable(iommu); + if (ret) + av1_iommu_detach_device(iommu->domain, dev); + + pm_runtime_put(iommu->dev); + + return ret; +} + +static void av1_iommu_domain_free(struct iommu_domain *domain) +{ + struct av1_iommu_domain *av1_domain = to_av1_domain(domain); + int i; + + WARN_ON(!list_empty(&av1_domain->iommus)); + + for (i = 0; i < NUM_DT_ENTRIES; i++) { + u32 dte = av1_domain->dt[i]; + + if (av1_dte_is_pt_valid(dte)) { + phys_addr_t pt_phys = av1_dte_pt_address(dte); + u32 *page_table = phys_to_virt(pt_phys); + + dma_unmap_single(dma_dev, pt_phys, + SPAGE_SIZE, DMA_TO_DEVICE); + free_page((unsigned long)page_table); + } + } + + dma_unmap_single(dma_dev, av1_domain->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); + free_page((unsigned long)av1_domain->dt); + + dma_unmap_single(dma_dev, av1_domain->pta_dma, + SPAGE_SIZE, DMA_TO_DEVICE); + free_page((unsigned long)av1_domain->pta); + + if (domain->type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&av1_domain->domain); + kfree(av1_domain); +} + +static struct iommu_device *av1_iommu_probe_device(struct device *dev) +{ + struct av1_iommudata *data; + struct av1_iommu *iommu; + + data = dev_iommu_priv_get(dev); + if (!data) + return ERR_PTR(-ENODEV); + + iommu = av1_iommu_from_dev(dev); + + pr_info("%s,%d, consumer : %s, supplier : %s\n", + __func__, __LINE__, dev_name(dev), dev_name(iommu->dev)); + + data->link = device_link_add(dev, iommu->dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + + /* set max segment size for dev, needed for single chunk map */ + if (!dev->dma_parms) + dev->dma_parms = kzalloc(sizeof(*dev->dma_parms), GFP_KERNEL); + if (!dev->dma_parms) + return ERR_PTR(-ENOMEM); + + dma_set_max_seg_size(dev, DMA_BIT_MASK(32)); + + return &iommu->iommu; +} + +static void av1_iommu_release_device(struct device *dev) +{ + struct av1_iommudata *data = dev_iommu_priv_get(dev); + + device_link_del(data->link); +} + +static struct iommu_group *av1_iommu_device_group(struct device *dev) +{ + struct av1_iommu *iommu; + + iommu = av1_iommu_from_dev(dev); + + return iommu_group_ref_get(iommu->group); +} + +static int av1_iommu_of_xlate(struct device *dev, + struct of_phandle_args *args) +{ + struct platform_device *iommu_dev; + struct av1_iommudata *data; + + data = devm_kzalloc(dma_dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + dev_info(dev, "%s,%d\n", __func__, __LINE__); + iommu_dev = of_find_device_by_node(args->np); + + data->iommu = platform_get_drvdata(iommu_dev); + + dev_iommu_priv_set(dev, data); + + platform_device_put(iommu_dev); + + return 0; +} + +static struct iommu_ops av1_iommu_ops = { + .domain_alloc = av1_iommu_domain_alloc, + .domain_free = av1_iommu_domain_free, + .attach_dev = av1_iommu_attach_device, + .detach_dev = av1_iommu_detach_device, + .map = av1_iommu_map, + .unmap = av1_iommu_unmap, + .flush_iotlb_all = av1_iommu_flush_tlb_all, + .probe_device = av1_iommu_probe_device, + .release_device = av1_iommu_release_device, + .iova_to_phys = av1_iommu_iova_to_phys, + .is_attach_deferred = av1_iommu_is_attach_deferred, + .device_group = av1_iommu_device_group, + .pgsize_bitmap = AV1_IOMMU_PGSIZE_BITMAP, + .of_xlate = av1_iommu_of_xlate, +}; + +static const struct of_device_id av1_iommu_dt_ids[] = { + { + .compatible = "rockchip,iommu-av1", + }, + { /* sentinel */ } +}; + +static int av1_iommu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct av1_iommu *iommu; + struct resource *res; + int num_res = pdev->num_resources; + int err, i; + const struct of_device_id *match; + + iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); + if (!iommu) + return -ENOMEM; + + match = of_match_device(av1_iommu_dt_ids, dev); + if (!match) + return -EINVAL; + + platform_set_drvdata(pdev, iommu); + iommu->dev = dev; + iommu->num_mmu = 0; + + iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases), + GFP_KERNEL); + if (!iommu->bases) + return -ENOMEM; + + for (i = 0; i < num_res; i++) { + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + if (!res) + continue; + iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(iommu->bases[i])) + continue; + iommu->num_mmu++; + } + if (iommu->num_mmu == 0) + return PTR_ERR(iommu->bases[0]); + + iommu->num_irq = platform_irq_count(pdev); + if (iommu->num_irq < 0) + return iommu->num_irq; + + err = devm_clk_bulk_get_all(dev, &iommu->clocks); + if (err >= 0) + iommu->num_clocks = err; + else if (err == -ENOENT) + iommu->num_clocks = 0; + else + return err; + + err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks); + if (err) + return err; + + iommu->group = iommu_group_alloc(); + if (IS_ERR(iommu->group)) { + err = PTR_ERR(iommu->group); + goto err_unprepare_clocks; + } + + err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev)); + if (err) + goto err_put_group; + + iommu_device_set_ops(&iommu->iommu, &av1_iommu_ops); + iommu_device_set_fwnode(&iommu->iommu, &dev->of_node->fwnode); + + err = iommu_device_register(&iommu->iommu); + if (err) + goto err_remove_sysfs; + + if (!dma_dev) + dma_dev = &pdev->dev; + + bus_set_iommu(&av1dec_bus, &av1_iommu_ops); + + pm_runtime_enable(dev); + + for (i = 0; i < iommu->num_irq; i++) { + int irq = platform_get_irq(pdev, i); + + if (irq < 0) + return irq; + + err = devm_request_irq(iommu->dev, irq, av1_iommu_irq, + IRQF_SHARED, dev_name(dev), iommu); + if (err) { + pm_runtime_disable(dev); + goto err_remove_sysfs; + } + } + + return 0; +err_remove_sysfs: + iommu_device_sysfs_remove(&iommu->iommu); +err_put_group: + iommu_group_put(iommu->group); +err_unprepare_clocks: + clk_bulk_unprepare(iommu->num_clocks, iommu->clocks); + return err; +} + +static void av1_iommu_shutdown(struct platform_device *pdev) +{ + struct av1_iommu *iommu = platform_get_drvdata(pdev); + int i; + + for (i = 0; i < iommu->num_irq; i++) { + int irq = platform_get_irq(pdev, i); + + devm_free_irq(iommu->dev, irq, iommu); + } + + pm_runtime_force_suspend(&pdev->dev); +} + +static int __maybe_unused av1_iommu_suspend(struct device *dev) +{ + struct av1_iommu *iommu = dev_get_drvdata(dev); + + if (!iommu->domain) + return 0; + + av1_iommu_disable(iommu); + return 0; +} + +static int __maybe_unused av1_iommu_resume(struct device *dev) +{ + struct av1_iommu *iommu = dev_get_drvdata(dev); + + if (!iommu->domain) + return 0; + + return av1_iommu_enable(iommu); +} + +static const struct dev_pm_ops av1_iommu_pm_ops = { + SET_RUNTIME_PM_OPS(av1_iommu_suspend, av1_iommu_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) +}; + +struct platform_driver rockchip_av1_iommu_driver = { + .probe = av1_iommu_probe, + .shutdown = av1_iommu_shutdown, + .driver = { + .name = "av1_iommu", + .of_match_table = av1_iommu_dt_ids, + .pm = &av1_iommu_pm_ops, + .suppress_bind_attrs = true, + }, +}; diff --git a/drivers/video/rockchip/mpp/mpp_service.c b/drivers/video/rockchip/mpp/mpp_service.c index cfab5fe163b7..e58a7df934fc 100644 --- a/drivers/video/rockchip/mpp/mpp_service.c +++ b/drivers/video/rockchip/mpp/mpp_service.c @@ -38,6 +38,7 @@ #define HAS_JPGDEC IS_ENABLED(CONFIG_ROCKCHIP_MPP_JPGDEC) #define HAS_RKVDEC2 IS_ENABLED(CONFIG_ROCKCHIP_MPP_RKVDEC2) #define HAS_RKVENC2 IS_ENABLED(CONFIG_ROCKCHIP_MPP_RKVENC2) +#define HAS_AV1DEC IS_ENABLED(CONFIG_ROCKCHIP_MPP_AV1DEC) #define MPP_REGISTER_DRIVER(srv, flag, X, x) {\ if (flag)\ @@ -97,7 +98,10 @@ static int mpp_add_driver(struct mpp_service *srv, &srv->grf_infos[type], grf_name); - ret = platform_driver_register(driver); + if (type == MPP_DRIVER_AV1DEC) + ret = av1dec_driver_register(driver); + else + ret = platform_driver_register(driver); if (ret) return ret; @@ -372,6 +376,7 @@ static int mpp_service_probe(struct platform_device *pdev) MPP_REGISTER_DRIVER(srv, HAS_JPGDEC, JPGDEC, jpgdec); MPP_REGISTER_DRIVER(srv, HAS_RKVDEC2, RKVDEC2, rkvdec2); MPP_REGISTER_DRIVER(srv, HAS_RKVENC2, RKVENC2, rkvenc2); + MPP_REGISTER_DRIVER(srv, HAS_AV1DEC, AV1DEC, av1dec); dev_info(dev, "probe success\n");