From e9490005011ecd63cdc5fc82d03d6443993dd887 Mon Sep 17 00:00:00 2001 From: Ding Wei Date: Wed, 26 Jan 2022 09:58:27 +0800 Subject: [PATCH] video: rockchip: mpp: rkvdec2: Add hard-ccu mode hard-ccu mode: 1. ccu hardware assign the tasks to each core. 2. if one core error, wait dule core idle, and reset dual core and ccu. 3. re-run tasks which have not finish. Change-Id: I3476c0de21de89019ce924122a85ab4352faafd0 Signed-off-by: Ding Wei --- drivers/video/rockchip/mpp/mpp_common.c | 14 +- drivers/video/rockchip/mpp/mpp_common.h | 2 + drivers/video/rockchip/mpp/mpp_rkvdec2.c | 79 +- drivers/video/rockchip/mpp/mpp_rkvdec2.h | 10 + drivers/video/rockchip/mpp/mpp_rkvdec2_link.c | 731 +++++++++++++++++- drivers/video/rockchip/mpp/mpp_rkvdec2_link.h | 23 + 6 files changed, 813 insertions(+), 46 deletions(-) diff --git a/drivers/video/rockchip/mpp/mpp_common.c b/drivers/video/rockchip/mpp/mpp_common.c index 043a5b3b41b3..3717b4fa83eb 100644 --- a/drivers/video/rockchip/mpp/mpp_common.c +++ b/drivers/video/rockchip/mpp/mpp_common.c @@ -2131,15 +2131,9 @@ int mpp_dev_probe(struct mpp_dev *mpp, return -ENODEV; } - if (mpp->task_capacity == 1) { - /* power domain autosuspend delay 2s */ - pm_runtime_set_autosuspend_delay(dev, 2000); - pm_runtime_use_autosuspend(dev); - } else { - dev_info(dev, "link mode task capacity %d\n", - mpp->task_capacity); - /* do not setup autosuspend on multi task device */ - } + /* power domain autosuspend delay 2s */ + pm_runtime_set_autosuspend_delay(dev, 2000); + pm_runtime_use_autosuspend(dev); kthread_init_work(&mpp->work, mpp_task_worker_default); @@ -2150,7 +2144,6 @@ int mpp_dev_probe(struct mpp_dev *mpp, device_init_wakeup(dev, true); pm_runtime_enable(dev); - mpp->irq = platform_get_irq(pdev, 0); if (mpp->irq < 0) { dev_err(dev, "No interrupt resource found\n"); @@ -2177,6 +2170,7 @@ int mpp_dev_probe(struct mpp_dev *mpp, ret = -ENOMEM; goto failed; } + mpp->io_base = res->start; /* * TODO: here or at the device itself, some device does not diff --git a/drivers/video/rockchip/mpp/mpp_common.h b/drivers/video/rockchip/mpp/mpp_common.h index f204d35a17dd..a4082ac4a4a8 100644 --- a/drivers/video/rockchip/mpp/mpp_common.h +++ b/drivers/video/rockchip/mpp/mpp_common.h @@ -348,6 +348,7 @@ struct mpp_dev { struct mpp_iommu_info *iommu_info; int (*fault_handler)(struct iommu_domain *iommu, struct device *iommu_dev, unsigned long iova, int status, void *arg); + resource_size_t io_base; atomic_t reset_request; atomic_t session_index; @@ -543,6 +544,7 @@ struct mpp_taskqueue { u32 core_id_max; u32 core_count; unsigned long dev_active_flags; + u32 iommu_fault; }; struct mpp_reset_group { diff --git a/drivers/video/rockchip/mpp/mpp_rkvdec2.c b/drivers/video/rockchip/mpp/mpp_rkvdec2.c index 780e2a364030..990561f1c0d5 100644 --- a/drivers/video/rockchip/mpp/mpp_rkvdec2.c +++ b/drivers/video/rockchip/mpp/mpp_rkvdec2.c @@ -307,7 +307,7 @@ void *rkvdec2_alloc_task(struct mpp_session *session, } static void *rkvdec2_rk3568_alloc_task(struct mpp_session *session, - struct mpp_task_msgs *msgs) + struct mpp_task_msgs *msgs) { u32 fmt; struct mpp_task *mpp_task = NULL; @@ -1349,15 +1349,26 @@ static int rkvdec2_ccu_probe(struct platform_device *pdev) struct rkvdec2_ccu *ccu; struct resource *res; struct device *dev = &pdev->dev; + u32 ccu_mode; ccu = devm_kzalloc(dev, sizeof(*ccu), GFP_KERNEL); if (!ccu) return -ENOMEM; ccu->dev = dev; + /* use task-level soft ccu default */ + ccu->ccu_mode = RKVDEC2_CCU_TASK_SOFT; atomic_set(&ccu->power_enabled, 0); + INIT_LIST_HEAD(&ccu->unused_list); + INIT_LIST_HEAD(&ccu->used_list); platform_set_drvdata(pdev, ccu); + if (!of_property_read_u32(dev->of_node, "rockchip,ccu-mode", &ccu_mode)) { + if (ccu_mode <= RKVDEC2_CCU_MODE_NULL || ccu_mode >= RKVDEC2_CCU_MODE_BUTT) + ccu_mode = RKVDEC2_CCU_TASK_SOFT; + ccu->ccu_mode = (enum RKVDEC2_CCU_MODE)ccu_mode; + } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ccu"); if (!res) { dev_err(dev, "no memory resource defined\n"); @@ -1370,12 +1381,6 @@ static int rkvdec2_ccu_probe(struct platform_device *pdev) return -ENODEV; } - device_init_wakeup(dev, true); - pm_runtime_enable(dev); - /* power domain autosuspend delay 2s */ - pm_runtime_set_autosuspend_delay(dev, 2000); - pm_runtime_use_autosuspend(dev); - ccu->aclk_info.clk = devm_clk_get(dev, "aclk_ccu"); if (!ccu->aclk_info.clk) mpp_err("failed on clk_get ccu aclk\n"); @@ -1386,6 +1391,13 @@ static int rkvdec2_ccu_probe(struct platform_device *pdev) else mpp_err("failed on clk_get ccu reset\n"); + /* power domain autosuspend delay 2s */ + pm_runtime_set_autosuspend_delay(dev, 2000); + pm_runtime_use_autosuspend(dev); + device_init_wakeup(dev, true); + pm_runtime_enable(dev); + + dev_info(dev, "ccu-mode: %d\n", ccu->ccu_mode); return 0; } @@ -1482,6 +1494,22 @@ static int rkvdec2_alloc_rcbbuf(struct platform_device *pdev, struct rkvdec2_dev if (!ret && dec->rcb_min_width) dev_info(dev, "min_width %u\n", dec->rcb_min_width); + /* if have, read rcb_info */ + dec->rcb_info_count = device_property_count_u32(dev, "rockchip,rcb-info"); + if (dec->rcb_info_count > 0 && + dec->rcb_info_count <= (sizeof(dec->rcb_infos) / sizeof(u32))) { + int i; + + ret = device_property_read_u32_array(dev, "rockchip,rcb-info", + dec->rcb_infos, dec->rcb_info_count); + if (!ret) { + dev_info(dev, "rcb_info_count %u\n", dec->rcb_info_count); + for (i = 0; i < dec->rcb_info_count; i += 2) + dev_info(dev, "[%u, %u]\n", + dec->rcb_infos[i], dec->rcb_infos[i+1]); + } + } + return 0; err_sram_map: @@ -1496,6 +1524,7 @@ static int rkvdec2_core_probe(struct platform_device *pdev) struct rkvdec2_dev *dec; struct mpp_dev *mpp; struct device *dev = &pdev->dev; + irq_handler_t irq_proc = NULL; dec = devm_kzalloc(dev, sizeof(*dec), GFP_KERNEL); if (!dec) @@ -1519,29 +1548,44 @@ static int rkvdec2_core_probe(struct platform_device *pdev) dev_err(dev, "probe sub driver failed\n"); return ret; } + dec->mmu_base = ioremap(dec->mpp.io_base + 0x600, 0x80); + if (!dec->mmu_base) + dev_err(dev, "mmu base map failed!\n"); + /* attach core to ccu */ ret = rkvdec2_attach_ccu(dev, dec); if (ret) { dev_err(dev, "attach ccu failed\n"); return ret; } - /* power domain autosuspend delay 2s */ - pm_runtime_set_autosuspend_delay(dev, 2000); - pm_runtime_use_autosuspend(dev); /* alloc rcb buffer */ rkvdec2_alloc_rcbbuf(pdev, dec); /* set device for link */ - rkvdec2_ccu_link_init(pdev, dec); + ret = rkvdec2_ccu_link_init(pdev, dec); + if (ret) + return ret; mpp->dev_ops->alloc_task = rkvdec2_ccu_alloc_task; - mpp->dev_ops->task_worker = rkvdec2_soft_ccu_worker; - kthread_init_work(&mpp->work, rkvdec2_soft_ccu_worker); + if (dec->ccu->ccu_mode == RKVDEC2_CCU_TASK_SOFT) { + mpp->dev_ops->task_worker = rkvdec2_soft_ccu_worker; + irq_proc = rkvdec2_soft_ccu_irq; + } else if (dec->ccu->ccu_mode == RKVDEC2_CCU_TASK_HARD) { + if (mpp->core_id == 0 && mpp->task_capacity > 1) { + dec->link_dec->task_capacity = mpp->task_capacity; + ret = rkvdec2_ccu_alloc_table(dec, dec->link_dec); + if (ret) + return ret; + } + mpp->dev_ops->task_worker = rkvdec2_hard_ccu_worker; + irq_proc = rkvdec2_hard_ccu_irq; + } + mpp->iommu_info->hdl = rkvdec2_ccu_iommu_fault_handle; + kthread_init_work(&mpp->work, mpp->dev_ops->task_worker); - mpp->fault_handler = rkvdec2_ccu_iommu_fault_handle; /* get irq request */ - ret = devm_request_threaded_irq(dev, mpp->irq, rkvdec2_soft_ccu_irq, NULL, + ret = devm_request_threaded_irq(dev, mpp->irq, irq_proc, NULL, IRQF_SHARED, dev_name(dev), mpp); if (ret) { dev_err(dev, "register interrupter runtime failed\n"); @@ -1667,7 +1711,10 @@ static int rkvdec2_remove(struct platform_device *pdev) struct rkvdec2_dev *dec = to_rkvdec2_dev(mpp); dev_info(dev, "remove device\n"); - + if (dec->mmu_base) { + iounmap(dec->mmu_base); + dec->mmu_base = NULL; + } rkvdec2_free_rcbbuf(pdev, dec); mpp_dev_remove(mpp); rkvdec2_procfs_remove(mpp); diff --git a/drivers/video/rockchip/mpp/mpp_rkvdec2.h b/drivers/video/rockchip/mpp/mpp_rkvdec2.h index be89535e6217..cc6bf19f355f 100644 --- a/drivers/video/rockchip/mpp/mpp_rkvdec2.h +++ b/drivers/video/rockchip/mpp/mpp_rkvdec2.h @@ -209,6 +209,8 @@ struct rkvdec2_dev { dma_addr_t rcb_iova; struct page *rcb_page; u32 rcb_min_width; + u32 rcb_info_count; + u32 rcb_infos[RKVDEC_MAX_RCB_NUM * 2]; /* for link mode */ struct rkvdec_link_dev *link_dec; @@ -218,6 +220,14 @@ struct rkvdec2_dev { struct rkvdec2_ccu *ccu; u32 core_mask; u32 task_index; + /* mmu info */ + void __iomem *mmu_base; + u32 fault_iova; + u32 mmu_fault; + u32 mmu0_st; + u32 mmu1_st; + u32 mmu0_pta; + u32 mmu1_pta; }; int mpp_set_rcbbuf(struct mpp_dev *mpp, struct mpp_session *session, diff --git a/drivers/video/rockchip/mpp/mpp_rkvdec2_link.c b/drivers/video/rockchip/mpp/mpp_rkvdec2_link.c index 41a4d9089aeb..77ce1401fde4 100644 --- a/drivers/video/rockchip/mpp/mpp_rkvdec2_link.c +++ b/drivers/video/rockchip/mpp/mpp_rkvdec2_link.c @@ -1567,7 +1567,6 @@ int rkvdec2_attach_ccu(struct device *dev, struct rkvdec2_dev *dec) struct device_node *np; struct platform_device *pdev; struct rkvdec2_ccu *ccu; - struct mpp_taskqueue *queue; mpp_debug_enter(); @@ -1590,10 +1589,11 @@ int rkvdec2_attach_ccu(struct device *dev, struct rkvdec2_dev *dec) dev_info(dev, "core_mask=%08x\n", dec->core_mask); /* if not the main-core, then attach the main core domain to current */ - queue = dec->mpp.queue; - if (&dec->mpp != queue->cores[0]) { + if (dec->mpp.core_id != 0) { + struct mpp_taskqueue *queue; struct mpp_iommu_info *ccu_info, *cur_info; + queue = dec->mpp.queue; /* set the ccu-domain for current device */ ccu_info = queue->cores[0]->iommu_info; cur_info = dec->mpp.iommu_info; @@ -1609,10 +1609,9 @@ int rkvdec2_attach_ccu(struct device *dev, struct rkvdec2_dev *dec) return 0; } -static void rkvdec2_ccu_link_timeout_work(struct work_struct *work_s) +static void rkvdec2_ccu_timeout_work(struct work_struct *work_s) { struct mpp_dev *mpp; - struct mpp_session *session; struct mpp_task *task = container_of(to_delayed_work(work_s), struct mpp_task, timeout_work); @@ -1626,15 +1625,8 @@ static void rkvdec2_ccu_link_timeout_work(struct work_struct *work_s) mpp_err("task %d session is null.\n", task->task_id); return; } - session = task->session; - - if (!session->mpp) { - mpp_err("task %d:%d mpp is null.\n", session->index, - task->task_id); - return; - } - mpp = task->mpp ? task->mpp : session->mpp; - mpp_err("task timeout\n"); + mpp = mpp_get_task_used_device(task, task->session); + mpp_err("%s, task timeout\n", dev_name(mpp->dev)); set_bit(TASK_STATE_TIMEOUT, &task->state); atomic_inc(&mpp->reset_request); atomic_inc(&mpp->queue->reset_request); @@ -1874,19 +1866,62 @@ void *rkvdec2_ccu_alloc_task(struct mpp_session *session, return &task->mpp_task; } +static void rkvdec2_ccu_check_pagefault_info(struct mpp_dev *mpp) +{ + u32 i = 0; + + for (i = 0; i < mpp->queue->core_count; i++) { + struct mpp_dev *core = mpp->queue->cores[i]; + struct rkvdec2_dev *dec = to_rkvdec2_dev(core); + void __iomem *mmu_base = dec->mmu_base; + u32 mmu0_st; + u32 mmu1_st; + u32 mmu0_pta; + u32 mmu1_pta; + + if (!mmu_base) + return; + + #define FAULT_STATUS 0x7e2 + rkvdec2_ccu_power_on(mpp->queue, dec->ccu); + + mmu0_st = readl(mmu_base + 0x4); + mmu1_st = readl(mmu_base + 0x44); + mmu0_pta = readl(mmu_base + 0xc); + mmu1_pta = readl(mmu_base + 0x4c); + + dec->mmu0_st = mmu0_st; + dec->mmu1_st = mmu1_st; + dec->mmu0_pta = mmu0_pta; + dec->mmu1_pta = mmu1_pta; + + pr_err("core %d mmu0 %08x %08x mm1 %08x %08x\n", + core->core_id, mmu0_st, mmu0_pta, mmu1_st, mmu1_pta); + if ((mmu0_st & FAULT_STATUS) || (mmu1_st & FAULT_STATUS) || + mmu0_pta || mmu1_pta) { + dec->fault_iova = readl(dec->link_dec->reg_base + 0x4); + dec->mmu_fault = 1; + pr_err("core %d fault iova %08x\n", core->core_id, dec->fault_iova); + rockchip_iommu_mask_irq(core->dev); + } else { + dec->mmu_fault = 0; + dec->fault_iova = 0; + } + } +} + int rkvdec2_ccu_iommu_fault_handle(struct iommu_domain *iommu, struct device *iommu_dev, unsigned long iova, int status, void *arg) { - u32 i = 0; struct mpp_dev *mpp = (struct mpp_dev *)arg; mpp_debug_enter(); - atomic_inc(&mpp->queue->reset_request); - for (i = 0; i < mpp->queue->core_count; i++) - rockchip_iommu_mask_irq(mpp->queue->cores[i]->dev); + rkvdec2_ccu_check_pagefault_info(mpp); + mpp->queue->iommu_fault = 1; + atomic_inc(&mpp->queue->reset_request); kthread_queue_work(&mpp->queue->worker, &mpp->work); mpp_debug_leave(); @@ -2041,6 +2076,36 @@ static bool rkvdec2_core_working(struct mpp_taskqueue *queue) return flag; } +static int rkvdec2_ccu_link_session_detach(struct mpp_dev *mpp, + struct mpp_taskqueue *queue) +{ + mutex_lock(&queue->session_lock); + while (atomic_read(&queue->detach_count)) { + struct mpp_session *session = NULL; + + session = list_first_entry_or_null(&queue->session_detach, + struct mpp_session, + session_link); + if (session) { + list_del_init(&session->session_link); + atomic_dec(&queue->detach_count); + } + + mutex_unlock(&queue->session_lock); + + if (session) { + mpp_dbg_session("%s detach count %d\n", dev_name(mpp->dev), + atomic_read(&queue->detach_count)); + mpp_session_deinit(session); + } + + mutex_lock(&queue->session_lock); + } + mutex_unlock(&queue->session_lock); + + return 0; +} + void rkvdec2_soft_ccu_worker(struct kthread_work *work_s) { struct mpp_task *mpp_task; @@ -2108,8 +2173,9 @@ get_task: mpp_debug(DEBUG_TASK_INFO, "pid %d, start hw %s\n", mpp_task->session->pid, dev_name(mpp->dev)); - INIT_DELAYED_WORK(&mpp_task->timeout_work, rkvdec2_ccu_link_timeout_work); - + set_bit(TASK_STATE_START, &mpp_task->state); + INIT_DELAYED_WORK(&mpp_task->timeout_work, rkvdec2_ccu_timeout_work); + schedule_delayed_work(&mpp_task->timeout_work, msecs_to_jiffies(WORK_TIMEOUT_MS)); rkvdec2_ccu_power_on(queue, dec->ccu); rkvdec2_soft_ccu_enqueue(mpp, mpp_task); @@ -2118,6 +2184,631 @@ done: list_empty(&queue->pending_list)) rkvdec2_ccu_power_off(queue, dec->ccu); out: + /* session detach out of queue */ + rkvdec2_ccu_link_session_detach(mpp, queue); + + mpp_debug_leave(); +} + +int rkvdec2_ccu_alloc_table(struct rkvdec2_dev *dec, + struct rkvdec_link_dev *link_dec) +{ + int ret, i; + struct mpp_dma_buffer *table; + struct mpp_dev *mpp = &dec->mpp; + + mpp_debug_enter(); + + /* alloc table pointer array */ + table = devm_kmalloc_array(mpp->dev, mpp->task_capacity, + sizeof(*table), GFP_KERNEL | __GFP_ZERO); + if (!table) + return -ENOMEM; + + /* alloc table buffer */ + ret = rkvdec2_link_alloc_table(mpp, link_dec); + if (ret) + return ret; + + /* init table array */ + dec->ccu->table_array = table; + for (i = 0; i < mpp->task_capacity; i++) { + table[i].iova = link_dec->table->iova + i * link_dec->link_node_size; + table[i].vaddr = link_dec->table->vaddr + i * link_dec->link_node_size; + table[i].size = link_dec->link_node_size; + INIT_LIST_HEAD(&table[i].link); + list_add_tail(&table[i].link, &dec->ccu->unused_list); + } + + return 0; +} + +static void rkvdec2_dump_ccu(struct rkvdec2_ccu *ccu) +{ + u32 i; + + for (i = 0; i < 10; i++) + mpp_err("ccu:reg[%d]=%08x\n", i, readl(ccu->reg_base + 4 * i)); + + for (i = 16; i < 22; i++) + mpp_err("ccu:reg[%d]=%08x\n", i, readl(ccu->reg_base + 4 * i)); +} + +static void rkvdec2_dump_link(struct rkvdec2_dev *dec) +{ + u32 i; + + for (i = 0; i < 10; i++) + mpp_err("link:reg[%d]=%08x\n", i, readl(dec->link_dec->reg_base + 4 * i)); +} + +static void rkvdec2_dump_core(struct mpp_dev *mpp, struct rkvdec2_task *task) +{ + u32 j; + + if (task) { + for (j = 0; j < 273; j++) + mpp_err("reg[%d]=%08x, %08x\n", j, mpp_read(mpp, j*4), task->reg[j]); + } else { + for (j = 0; j < 273; j++) + mpp_err("reg[%d]=%08x\n", j, mpp_read(mpp, j*4)); + } +} + +irqreturn_t rkvdec2_hard_ccu_irq(int irq, void *param) +{ + u32 irq_status; + struct mpp_dev *mpp = param; + struct rkvdec2_dev *dec = to_rkvdec2_dev(mpp); + + irq_status = readl(dec->link_dec->reg_base + RKVDEC_LINK_IRQ_BASE); + dec->ccu->ccu_core_work_mode = readl(dec->ccu->reg_base + RKVDEC_CCU_CORE_WORK_BASE); + if (irq_status & RKVDEC_LINK_BIT_IRQ_RAW) { + dec->link_dec->irq_status = irq_status; + mpp->irq_status = mpp_read(mpp, RKVDEC_REG_INT_EN); + mpp_debug(DEBUG_IRQ_STATUS, "core %d link_irq=%08x, core_irq=%08x\n", + mpp->core_id, irq_status, mpp->irq_status); + + writel(irq_status & 0xfffff0ff, + dec->link_dec->reg_base + RKVDEC_LINK_IRQ_BASE); + + kthread_queue_work(&mpp->queue->worker, &mpp->work); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int rkvdec2_hard_ccu_finish(struct rkvdec_link_info *hw, struct rkvdec2_task *task) +{ + u32 i, off, s, n; + struct rkvdec_link_part *part = hw->part_r; + u32 *tb_reg = (u32 *)task->table->vaddr; + + mpp_debug_enter(); + + for (i = 0; i < hw->part_r_num; i++) { + off = part[i].tb_reg_off; + s = part[i].reg_start; + n = part[i].reg_num; + memcpy(&task->reg[s], &tb_reg[off], n * sizeof(u32)); + } + /* revert hack for irq status */ + task->reg[RKVDEC_REG_INT_EN_INDEX] = task->irq_status; + + mpp_debug_leave(); + + return 0; +} + +static int rkvdec2_hard_ccu_dequeue(struct mpp_taskqueue *queue, + struct rkvdec2_ccu *ccu, + struct rkvdec_link_info *hw) +{ + struct mpp_task *mpp_task = NULL, *n; + u32 dump_reg = 0; + u32 dequeue_none = 0; + + mpp_debug_enter(); + list_for_each_entry_safe(mpp_task, n, &queue->running_list, queue_link) { + u32 timeout_flag = test_bit(TASK_STATE_TIMEOUT, &mpp_task->state); + u32 abort_flag = test_bit(TASK_STATE_ABORT, &mpp_task->state); + struct rkvdec2_task *task = to_rkvdec2_task(mpp_task); + u32 *tb_reg = (u32 *)task->table->vaddr; + u32 irq_status = tb_reg[hw->tb_reg_int]; + u32 ccu_decoded_num, ccu_total_dec_num; + + ccu_decoded_num = readl(ccu->reg_base + RKVDEC_CCU_DEC_NUM_BASE); + ccu_total_dec_num = readl(ccu->reg_base + RKVDEC_CCU_TOTAL_NUM_BASE); + mpp_debug(DEBUG_IRQ_CHECK, + "session %d task %d w:h[%d %d] err %d irq_status %08x timeout=%u abort=%u iova %08x next %08x ccu[%d %d]\n", + mpp_task->session->index, mpp_task->task_index, task->width, + task->height, !!(irq_status & RKVDEC_INT_ERROR_MASK), irq_status, + timeout_flag, abort_flag, (u32)task->table->iova, + ((u32 *)task->table->vaddr)[hw->tb_reg_next], + ccu_decoded_num, ccu_total_dec_num); + + if (irq_status || timeout_flag || abort_flag) { + set_bit(TASK_STATE_HANDLE, &mpp_task->state); + cancel_delayed_work(&mpp_task->timeout_work); + mpp_time_diff(mpp_task); + task->irq_status = irq_status; + + if (irq_status) + rkvdec2_hard_ccu_finish(hw, task); + + set_bit(TASK_STATE_FINISH, &mpp_task->state); + set_bit(TASK_STATE_DONE, &mpp_task->state); + + if (timeout_flag && !dump_reg && mpp_debug_unlikely(DEBUG_DUMP_ERR_REG)) { + u32 i; + + mpp_err("###### ccu #####\n"); + rkvdec2_dump_ccu(ccu); + for (i = 0; i < queue->core_count; i++) { + mpp_err("###### core %d #####\n", i); + rkvdec2_dump_link(to_rkvdec2_dev(queue->cores[i])); + rkvdec2_dump_core(queue->cores[i], task); + } + dump_reg = 1; + } + list_move_tail(&task->table->link, &ccu->unused_list); + /* free task */ + list_del_init(&mpp_task->queue_link); + /* Wake up the GET thread */ + wake_up(&mpp_task->wait); + if ((irq_status & RKVDEC_INT_ERROR_MASK) || timeout_flag) { + pr_err("session %d task %d irq_status %08x timeout=%u abort=%u\n", + mpp_task->session->index, mpp_task->task_index, + irq_status, timeout_flag, abort_flag); + atomic_inc(&queue->reset_request); + } + + kref_put(&mpp_task->ref, mpp_free_task); + } else { + dequeue_none++; + /* + * there are only 2 cores, + * if dequeue not finish task more than 2, + * means the others task still not get run by hw, can break early. + */ + if (dequeue_none > 2) + break; + } + } + + mpp_debug_leave(); + return 0; +} + +static int rkvdec2_hard_ccu_reset(struct mpp_taskqueue *queue, struct rkvdec2_ccu *ccu) +{ + int i = 0; + + mpp_debug_enter(); + + /* reset and active core */ + for (i = 0; i < queue->core_count; i++) { + u32 val = 0; + struct mpp_dev *mpp = queue->cores[i]; + struct rkvdec2_dev *dec = to_rkvdec2_dev(mpp); + + if (mpp->disable) + continue; + dev_info(mpp->dev, "resetting...\n"); + disable_hardirq(mpp->irq); + /* force idle */ + writel(dec->core_mask, ccu->reg_base + RKVDEC_CCU_CORE_IDLE_BASE); + writel(0, ccu->reg_base + RKVDEC_CCU_WORK_BASE); + + { + /* soft reset */ + u32 val; + + mpp_write(mpp, RKVDEC_REG_IMPORTANT_BASE, RKVDEC_SOFTREST_EN); + udelay(5); + val = mpp_read(mpp, RKVDEC_REG_INT_EN); + if (!(val & RKVDEC_SOFT_RESET_READY)) + mpp_err("soft reset fail, int %08x\n", val); + + // /* cru reset */ + // dev_info(mpp->dev, "cru reset\n"); + // rkvdec2_reset(mpp); + } +#if IS_ENABLED(CONFIG_ROCKCHIP_SIP) + rockchip_dmcfreq_lock(); + sip_smc_vpu_reset(i, 0, 0); + rockchip_dmcfreq_unlock(); +#else + rkvdec2_reset(mpp); +#endif + mpp_iommu_refresh(mpp->iommu_info, mpp->dev); + enable_irq(mpp->irq); + atomic_set(&mpp->reset_request, 0); + val = mpp_read_relaxed(mpp, 272*4); + dev_info(mpp->dev, "reset done, idle %d\n", (val & 1)); + } + /* reset ccu */ + mpp_safe_reset(ccu->rst_a); + udelay(5); + mpp_safe_unreset(ccu->rst_a); + + mpp_debug_leave(); + return 0; +} + +#define RKVDEC2_1080P_PIXELS (1920*1080) +#define RKVDEC2_4K_PIXELS (4096*2304) +#define RKVDEC2_8K_PIXELS (7680*4320) +#define RKVDEC2_TIMEOUT_20MS (0xefffff) +#define RKVDEC2_TIMEOUT_50MS (0x2cfffff) +#define RKVDEC2_TIMEOUT_100MS (0x4ffffff) + +static u32 rkvdec2_get_timeout_threshold(struct rkvdec2_task *task) +{ + u32 pixels = task->pixels; + + if (pixels < RKVDEC2_1080P_PIXELS) + return RKVDEC2_TIMEOUT_20MS; + else if (pixels < RKVDEC2_4K_PIXELS) + return RKVDEC2_TIMEOUT_50MS; + else + return RKVDEC2_TIMEOUT_100MS; +} + +static struct mpp_task * +rkvdec2_hard_ccu_prepare(struct mpp_task *mpp_task, + struct rkvdec2_ccu *ccu, struct rkvdec_link_info *hw) +{ + u32 i, off, s, n; + u32 *tb_reg; + struct mpp_dma_buffer *table = NULL; + struct rkvdec_link_part *part; + struct rkvdec2_task *task = to_rkvdec2_task(mpp_task); + + mpp_debug_enter(); + + if (test_bit(TASK_STATE_PREPARE, &mpp_task->state)) + return mpp_task; + + /* ensure that cur table iova points to the next link table*/ + { + struct mpp_dma_buffer *table0 = NULL, *table1 = NULL, *n; + + list_for_each_entry_safe(table, n, &ccu->unused_list, link) { + if (!table0) { + table0 = table; + continue; + } + if (!table1) + table1 = table; + break; + } + if (!table0 || !table1) + return NULL; + ((u32 *)table0->vaddr)[hw->tb_reg_next] = table1->iova; + table = table0; + } + + /* set session idx */ + rkvdec2_set_core_info(task->reg, mpp_task->session->index); + tb_reg = (u32 *)table->vaddr; + part = hw->part_w; + + /* disable multicore pu/colmv offset req timeout reset */ + task->reg[RKVDEC_REG_EN_MODE_SET] |= BIT(1); + task->reg[RKVDEC_REG_TIMEOUT_THRESHOLD] = rkvdec2_get_timeout_threshold(task); + + for (i = 0; i < hw->part_w_num; i++) { + off = part[i].tb_reg_off; + s = part[i].reg_start; + n = part[i].reg_num; + memcpy(&tb_reg[off], &task->reg[s], n * sizeof(u32)); + } + + /* memset read registers */ + part = hw->part_r; + for (i = 0; i < hw->part_r_num; i++) { + off = part[i].tb_reg_off; + n = part[i].reg_num; + memset(&tb_reg[off], 0, n * sizeof(u32)); + } + list_move_tail(&table->link, &ccu->used_list); + task->table = table; + set_bit(TASK_STATE_PREPARE, &mpp_task->state); + mpp_dbg_ccu("session %d task %d iova %08x next %08x\n", + mpp_task->session->index, mpp_task->task_index, (u32)task->table->iova, + ((u32 *)task->table->vaddr)[hw->tb_reg_next]); + + mpp_debug_leave(); + + return mpp_task; +} + +static int rkvdec2_ccu_link_fix_rcb_regs(struct rkvdec2_dev *dec) +{ + int ret = 0; + u32 i, val; + u32 reg, reg_idx, rcb_size, rcb_offset; + + if (!dec->rcb_iova && !dec->rcb_info_count) + goto done; + /* check whether fixed */ + val = readl(dec->link_dec->reg_base + RKVDEC_LINK_IRQ_BASE); + if (val & RKVDEC_CCU_BIT_FIX_RCB) + goto done; + /* set registers */ + rcb_offset = 0; + for (i = 0; i < dec->rcb_info_count; i += 2) { + reg_idx = dec->rcb_infos[i]; + rcb_size = dec->rcb_infos[i + 1]; + mpp_debug(DEBUG_SRAM_INFO, + "rcb: reg %u size %u offset %u sram_size %u rcb_size %u\n", + reg_idx, rcb_size, rcb_offset, dec->sram_size, dec->rcb_size); + if ((rcb_offset + rcb_size) > dec->rcb_size) { + mpp_err("rcb: reg[%u] set failed.\n", reg_idx); + ret = -ENOMEM; + goto done; + } + reg = dec->rcb_iova + rcb_offset; + mpp_write(&dec->mpp, reg_idx * sizeof(u32), reg); + rcb_offset += rcb_size; + } + + val |= RKVDEC_CCU_BIT_FIX_RCB; + writel(val, dec->link_dec->reg_base + RKVDEC_LINK_IRQ_BASE); +done: + return ret; +} + +static int rkvdec2_hard_ccu_enqueue(struct rkvdec2_ccu *ccu, + struct mpp_task *mpp_task, + struct mpp_taskqueue *queue, + struct mpp_dev *mpp) +{ + u32 ccu_en, work_mode, link_mode; + struct rkvdec2_task *task = to_rkvdec2_task(mpp_task); + + mpp_debug_enter(); + + if (test_bit(TASK_STATE_START, &mpp_task->state)) + goto done; + + ccu_en = readl(ccu->reg_base + RKVDEC_CCU_WORK_BASE); + mpp_dbg_ccu("ccu_en=%d\n", ccu_en); + if (!ccu_en) { + u32 i; + + /* set work mode */ + work_mode = 0; + for (i = 0; i < queue->core_count; i++) { + u32 val; + struct mpp_dev *core = queue->cores[i]; + struct rkvdec2_dev *dec = to_rkvdec2_dev(core); + + if (mpp->disable) + continue; + work_mode |= dec->core_mask; + rkvdec2_ccu_link_fix_rcb_regs(dec); + /* control by ccu */ + val = readl(dec->link_dec->reg_base + RKVDEC_LINK_IRQ_BASE); + val |= RKVDEC_LINK_BIT_CCU_WORK_MODE; + writel(val, dec->link_dec->reg_base + RKVDEC_LINK_IRQ_BASE); + } + writel(work_mode, ccu->reg_base + RKVDEC_CCU_CORE_WORK_BASE); + ccu->ccu_core_work_mode = readl(ccu->reg_base + RKVDEC_CCU_CORE_WORK_BASE); + mpp_dbg_ccu("ccu_work_mode=%08x, ccu_work_status=%08x\n", + readl(ccu->reg_base + RKVDEC_CCU_CORE_WORK_BASE), + readl(ccu->reg_base + RKVDEC_CCU_CORE_STA_BASE)); + + /* set auto gating */ + writel(RKVDEC_CCU_BIT_AUTOGATE, ccu->reg_base + RKVDEC_CCU_CTRL_BASE); + /* link start base */ + writel(task->table->iova, ccu->reg_base + RKVDEC_CCU_CFG_ADDR_BASE); + /* enable link */ + writel(RKVDEC_CCU_BIT_WORK_EN, ccu->reg_base + RKVDEC_CCU_WORK_BASE); + } + + /* set link mode */ + link_mode = ccu_en ? RKVDEC_CCU_BIT_ADD_MODE : 0; + writel(link_mode | RKVDEC_LINK_ADD_CFG_NUM, ccu->reg_base + RKVDEC_CCU_LINK_MODE_BASE); + + /* flush tlb before starting hardware */ + mpp_iommu_flush_tlb(mpp->iommu_info); + /* wmb */ + wmb(); + /* configure done */ + writel(RKVDEC_CCU_BIT_CFG_DONE, ccu->reg_base + RKVDEC_CCU_CFG_DONE_BASE); + + mpp_time_record(mpp_task); + set_bit(TASK_STATE_START, &mpp_task->state); + INIT_DELAYED_WORK(&mpp_task->timeout_work, rkvdec2_ccu_timeout_work); + schedule_delayed_work(&mpp_task->timeout_work, msecs_to_jiffies(WORK_TIMEOUT_MS)); + /* pending to running */ + if (!test_bit(TASK_STATE_RUNNING, &mpp_task->state)) { + mutex_lock(&queue->pending_lock); + set_bit(TASK_STATE_RUNNING, &mpp_task->state); + list_move_tail(&mpp_task->queue_link, &queue->running_list); + mutex_unlock(&queue->pending_lock); + } + mpp_dbg_ccu("session %d task %d iova=%08x task->state=%lx link_mode=%08x\n", + mpp_task->session->index, mpp_task->task_index, + (u32)task->table->iova, mpp_task->state, + readl(ccu->reg_base + RKVDEC_CCU_LINK_MODE_BASE)); +done: + mpp_debug_leave(); + + return 0; +} + +static void rkvdec2_hard_ccu_handle_pagefault_task(struct rkvdec2_dev *dec, + struct mpp_task *mpp_task) +{ + struct rkvdec2_task *task = to_rkvdec2_task(mpp_task); + + mpp_dbg_ccu("session %d task %d w:h[%d %d] pagefault mmu0[%08x %08x] mmu1[%08x %08x] fault_iova %08x\n", + mpp_task->session->index, mpp_task->task_index, + task->width, task->height, dec->mmu0_st, dec->mmu0_pta, + dec->mmu1_st, dec->mmu1_pta, dec->fault_iova); + + set_bit(TASK_STATE_HANDLE, &mpp_task->state); + task->irq_status |= BIT(4); + cancel_delayed_work(&mpp_task->timeout_work); + rkvdec2_hard_ccu_finish(dec->link_dec->info, task); + set_bit(TASK_STATE_FINISH, &mpp_task->state); + set_bit(TASK_STATE_DONE, &mpp_task->state); + list_move_tail(&task->table->link, &dec->ccu->unused_list); + list_del_init(&mpp_task->queue_link); + /* Wake up the GET thread */ + wake_up(&mpp_task->wait); + kref_put(&mpp_task->ref, mpp_free_task); + dec->mmu_fault = 0; + dec->fault_iova = 0; +} + +static void rkvdec2_hard_ccu_pagefault_proc(struct mpp_taskqueue *queue) +{ + struct mpp_task *loop = NULL, *n; + + list_for_each_entry_safe(loop, n, &queue->running_list, queue_link) { + struct rkvdec2_task *task = to_rkvdec2_task(loop); + u32 iova = (u32)task->table->iova; + u32 i; + + for (i = 0; i < queue->core_count; i++) { + struct mpp_dev *core = queue->cores[i]; + struct rkvdec2_dev *dec = to_rkvdec2_dev(core); + + if (!dec->mmu_fault || dec->fault_iova != iova) + continue; + rkvdec2_hard_ccu_handle_pagefault_task(dec, loop); + } + } +} + +static void rkvdec2_hard_ccu_resend_tasks(struct mpp_dev *mpp, struct mpp_taskqueue *queue) +{ + struct rkvdec2_task *task_pre = NULL; + struct mpp_task *loop = NULL, *n; + struct rkvdec2_dev *dec = to_rkvdec2_dev(mpp); + + /* re sort running list */ + list_for_each_entry_safe(loop, n, &queue->running_list, queue_link) { + struct rkvdec2_task *task = to_rkvdec2_task(loop); + u32 *tb_reg = (u32 *)task->table->vaddr; + u32 irq_status = tb_reg[dec->link_dec->info->tb_reg_int]; + + if (!irq_status) { + if (task_pre) { + tb_reg = (u32 *)task_pre->table->vaddr; + tb_reg[dec->link_dec->info->tb_reg_next] = task->table->iova; + } + task_pre = task; + } + } + + if (task_pre) { + struct mpp_dma_buffer *tbl; + u32 *tb_reg; + + tbl = list_first_entry_or_null(&dec->ccu->unused_list, + struct mpp_dma_buffer, link); + WARN_ON(!tbl); + if (tbl) { + tb_reg = (u32 *)task_pre->table->vaddr; + tb_reg[dec->link_dec->info->tb_reg_next] = tbl->iova; + } + } + + /* resend */ + list_for_each_entry_safe(loop, n, &queue->running_list, queue_link) { + struct rkvdec2_task *task = to_rkvdec2_task(loop); + u32 *tb_reg = (u32 *)task->table->vaddr; + u32 irq_status = tb_reg[dec->link_dec->info->tb_reg_int]; + + mpp_dbg_ccu("reback: session %d task %d iova %08x next %08x irq_status 0x%08x\n", + loop->session->index, loop->task_index, (u32)task->table->iova, + tb_reg[dec->link_dec->info->tb_reg_next], irq_status); + + if (!irq_status) { + cancel_delayed_work(&loop->timeout_work); + clear_bit(TASK_STATE_START, &loop->state); + rkvdec2_hard_ccu_enqueue(dec->ccu, loop, queue, mpp); + } + } +} + +void rkvdec2_hard_ccu_worker(struct kthread_work *work_s) +{ + struct mpp_task *mpp_task; + struct mpp_dev *mpp = container_of(work_s, struct mpp_dev, work); + struct mpp_taskqueue *queue = mpp->queue; + struct rkvdec2_dev *dec = to_rkvdec2_dev(mpp); + + mpp_debug_enter(); + + /* process all finished task in running list */ + rkvdec2_hard_ccu_dequeue(queue, dec->ccu, dec->link_dec->info); + + /* process reset request */ + if (atomic_read(&queue->reset_request) && + (list_empty(&queue->running_list) || !dec->ccu->ccu_core_work_mode)) { + /* + * cancel running list timeout work to avoid + * sw timeout causeby reset long time + */ + struct mpp_task *loop = NULL, *n; + + list_for_each_entry_safe(loop, n, &queue->running_list, queue_link) { + cancel_delayed_work(&loop->timeout_work); + } + /* reset process */ + rkvdec2_hard_ccu_reset(queue, dec->ccu); + atomic_set(&queue->reset_request, 0); + /* if iommu pagefault, find the fault task and drop it */ + if (queue->iommu_fault) { + rkvdec2_hard_ccu_pagefault_proc(queue); + queue->iommu_fault = 0; + } + + /* relink running task iova in list, and resend them to hw */ + if (!list_empty(&queue->running_list)) + rkvdec2_hard_ccu_resend_tasks(mpp, queue); + } +get_task: + /* get one task form pending list */ + mutex_lock(&queue->pending_lock); + mpp_task = list_first_entry_or_null(&queue->pending_list, + struct mpp_task, queue_link); + mutex_unlock(&queue->pending_lock); + + if (!mpp_task) + goto done; + if (test_bit(TASK_STATE_ABORT, &mpp_task->state)) { + mutex_lock(&queue->pending_lock); + list_del_init(&mpp_task->queue_link); + mutex_unlock(&queue->pending_lock); + kref_put(&mpp_task->ref, mpp_free_task); + goto get_task; + } + + if (atomic_read(&queue->reset_request)) + mpp_task = NULL; + else + mpp_task = rkvdec2_hard_ccu_prepare(mpp_task, dec->ccu, dec->link_dec->info); + + if (!mpp_task) + goto done; + + rkvdec2_ccu_power_on(queue, dec->ccu); + rkvdec2_hard_ccu_enqueue(dec->ccu, mpp_task, queue, mpp); +done: + mutex_lock(&queue->pending_lock); + if (list_empty(&queue->running_list) && + list_empty(&queue->pending_list)) + rkvdec2_ccu_power_off(queue, dec->ccu); + mutex_unlock(&queue->pending_lock); + /* session detach out of queue */ mpp_session_cleanup_detach(queue, work_s); diff --git a/drivers/video/rockchip/mpp/mpp_rkvdec2_link.h b/drivers/video/rockchip/mpp/mpp_rkvdec2_link.h index 270cd46a71e7..053bc6ae86bb 100644 --- a/drivers/video/rockchip/mpp/mpp_rkvdec2_link.h +++ b/drivers/video/rockchip/mpp/mpp_rkvdec2_link.h @@ -18,10 +18,14 @@ #define RKVDEC_REG_SECOND_EN_INDEX 12 #define RKVDEC_WAIT_RESET_EN BIT(7) +#define RKVDEC_REG_EN_MODE_SET 13 + #define RKVDEC_REG_DEBUG_INT_BASE 0x440 #define RKVDEC_REG_DEBUG_INT_INDEX 272 #define RKVDEC_BIT_BUS_IDLE BIT(0) +#define RKVDEC_REG_TIMEOUT_THRESHOLD 32 + /* define for link hardware */ #define RKVDEC_LINK_ADD_CFG_NUM 1 @@ -181,6 +185,13 @@ struct rkvdec_link_dev { u32 stuff_cnt; }; +enum RKVDEC2_CCU_MODE { + RKVDEC2_CCU_MODE_NULL = 0, + RKVDEC2_CCU_TASK_SOFT = 1, + RKVDEC2_CCU_TASK_HARD = 2, + RKVDEC2_CCU_MODE_BUTT, +}; + struct rkvdec2_ccu { struct device *dev; /* register base */ @@ -192,6 +203,13 @@ struct rkvdec2_ccu { struct proc_dir_entry *procfs; #endif struct reset_control *rst_a; + enum RKVDEC2_CCU_MODE ccu_mode; + u32 ccu_core_work_mode; + + struct mpp_dma_buffer *table_array; + struct list_head unused_list; + struct list_head used_list; + u32 timeout_flag; }; extern struct rkvdec_link_info rkvdec_link_rk356x_hw_info; @@ -222,4 +240,9 @@ int rkvdec2_ccu_iommu_fault_handle(struct iommu_domain *iommu, irqreturn_t rkvdec2_soft_ccu_irq(int irq, void *param); void rkvdec2_soft_ccu_worker(struct kthread_work *work_s); +int rkvdec2_ccu_alloc_table(struct rkvdec2_dev *dec, + struct rkvdec_link_dev *link_dec); +irqreturn_t rkvdec2_hard_ccu_irq(int irq, void *param); +void rkvdec2_hard_ccu_worker(struct kthread_work *work_s); + #endif