diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi index 1f4e4363c5a0..ed49b541fd1e 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi @@ -923,6 +923,10 @@ status = "okay"; }; +&rkvenc_ccu { + status = "okay"; +}; + &rkvenc0 { status = "okay"; }; @@ -931,6 +935,14 @@ status = "okay"; }; +&rkvenc1 { + status = "okay"; +}; + +&rkvenc1_mmu { + status = "okay"; +}; + &saradc { status = "okay"; vref-supply = <&vcc_1v8_s0>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-nvr.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-nvr.dtsi index af0ea3af82ae..a8df08c4e58f 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-nvr.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-nvr.dtsi @@ -167,6 +167,10 @@ status = "okay"; }; +&rkvenc_ccu { + status = "okay"; +}; + &rkvenc0 { status = "okay"; }; @@ -175,6 +179,14 @@ status = "okay"; }; +&rkvenc1 { + status = "okay"; +}; + +&rkvenc1_mmu { + status = "okay"; +}; + &saradc { status = "okay"; vref-supply = <&vcc_1v8_cam_s0>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-evb.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s-evb.dtsi index 416ccba88bf3..8bc51f6b2811 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588s-evb.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588s-evb.dtsi @@ -922,6 +922,10 @@ status = "okay"; }; +&rkvenc_ccu { + status = "okay"; +}; + &rkvenc0 { status = "okay"; }; @@ -930,6 +934,14 @@ status = "okay"; }; +&rkvenc1 { + status = "okay"; +}; + +&rkvenc1_mmu { + status = "okay"; +}; + &saradc { status = "okay"; vref-supply = <&vcc_1v8_s0>; diff --git a/arch/arm64/boot/dts/rockchip/rk3588s.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s.dtsi index f26f3543a39f..94c96038be60 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588s.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588s.dtsi @@ -39,6 +39,8 @@ rkcif_mipi_lvds1= &rkcif_mipi_lvds1; rkcif_mipi_lvds2= &rkcif_mipi_lvds2; rkcif_mipi_lvds3= &rkcif_mipi_lvds3; + rkvenc0 = &rkvenc0; + rkvenc1 = &rkvenc1; serial0 = &uart0; serial1 = &uart1; serial2 = &uart2; @@ -2026,8 +2028,8 @@ status = "disabled"; }; - rkvenc0: rkvenc@fdbd0000 { - compatible = "rockchip,rkv-encoder-v2"; + rkvenc0: rkvenc-core@fdbd0000 { + compatible = "rockchip,rkv-encoder-v2-core"; reg = <0x0 0xfdbd0000 0x0 0x6000>; interrupts = ; interrupt-names = "irq_rkvenc0"; @@ -2063,7 +2065,7 @@ status = "disabled"; }; - rkvenc_core1: rkvenc@fdbe0000 { + rkvenc1: rkvenc-core@fdbe0000 { compatible = "rockchip,rkv-encoder-v2-core"; reg = <0x0 0xfdbe0000 0x0 0x6000>; interrupts = ; @@ -2078,7 +2080,7 @@ iommus = <&rkvenc1_mmu>; rockchip,srv = <&mpp_srv>; rockchip,ccu = <&rkvenc_ccu>; - rockchip,taskqueue-node = <8>; + rockchip,taskqueue-node = <7>; rockchip,task-capacity = <8>; power-domains = <&power RK3588_PD_VENC1>; status = "disabled"; diff --git a/drivers/video/rockchip/mpp/mpp_common.c b/drivers/video/rockchip/mpp/mpp_common.c index 2fa4b87a2949..37c8432a5aab 100644 --- a/drivers/video/rockchip/mpp/mpp_common.c +++ b/drivers/video/rockchip/mpp/mpp_common.c @@ -398,18 +398,20 @@ static void mpp_free_task(struct kref *ref) } session = task->session; - mpp_debug_func(DEBUG_TASK_INFO, - "session=%p, task=%p, state=0x%lx, abort_request=%d\n", - session, task, task->state, + mpp_debug_func(DEBUG_TASK_INFO, "task %d:%d state 0x%lx abort %d\n", + session->index, task->task_index, task->state, atomic_read(&task->abort_request)); - if (!session->mpp) { - mpp_err("session %p, session->mpp is null.\n", session); + + mpp = task->mpp ? task->mpp : session->mpp; + if (!mpp) { + mpp_err("task %d:%d mpp is null.\n", + session->index, task->task_index); return; } - mpp = session->mpp; if (mpp->dev_ops->free_task) mpp->dev_ops->free_task(session, task); + /* Decrease reference count */ atomic_dec(&session->task_count); atomic_dec(&mpp->task_count); @@ -581,7 +583,6 @@ int mpp_dev_reset(struct mpp_dev *mpp) mpp_iommu_down_write(mpp->iommu_info); mpp_reset_down_write(mpp->reset_group); atomic_set(&mpp->reset_request, 0); - mpp_iommu_detach(mpp->iommu_info); rockchip_save_qos(mpp->dev); if (mpp->hw_ops->reset) @@ -594,7 +595,6 @@ int mpp_dev_reset(struct mpp_dev *mpp) */ mpp_iommu_refresh(mpp->iommu_info, mpp->dev); - mpp_iommu_attach(mpp->iommu_info); mpp_reset_up_write(mpp->reset_group); mpp_iommu_up_write(mpp->iommu_info); @@ -623,15 +623,6 @@ static int mpp_task_run(struct mpp_dev *mpp, } else { mpp_set_grf(mpp->grf_info); } - /* - * for iommu share hardware, should attach to ensure - * working in current device - */ - ret = mpp_iommu_attach(mpp->iommu_info); - if (ret) { - dev_err(mpp->dev, "mpp_iommu_attach failed\n"); - return -ENODATA; - } mpp_power_on(mpp); mpp_time_record(task); @@ -697,10 +688,12 @@ static void mpp_task_worker_default(struct kthread_work *work_s) */ /* Push a pending task to running queue */ if (task) { + struct mpp_dev *task_mpp = task->mpp ? task->mpp : mpp; + mpp_taskqueue_pending_to_run(queue, task); set_bit(TASK_STATE_RUNNING, &task->state); - if (mpp_task_run(mpp, task)) - mpp_taskqueue_pop_running(mpp->queue, task); + if (mpp_task_run(task_mpp, task)) + mpp_taskqueue_pop_running(queue, task); } done: @@ -881,12 +874,43 @@ struct mpp_taskqueue *mpp_taskqueue_init(struct device *dev) static void mpp_attach_workqueue(struct mpp_dev *mpp, struct mpp_taskqueue *queue) { - mpp->queue = queue; + s32 core_id; + INIT_LIST_HEAD(&mpp->queue_link); + mutex_lock(&queue->dev_lock); + + if (mpp->core_id >= 0) + core_id = mpp->core_id; + else + core_id = queue->core_count; + + if (core_id < 0 || core_id >= MPP_MAX_CORE_NUM) { + dev_err(mpp->dev, "invalid core id %d\n", core_id); + goto done; + } + + if (queue->cores[core_id]) { + dev_err(mpp->dev, "can not attach device with same id %d", core_id); + goto done; + } + + queue->cores[core_id] = mpp; + queue->core_count++; + + set_bit(core_id, &queue->core_idle); list_add_tail(&mpp->queue_link, &queue->dev_list); + + mpp->core_id = core_id; + mpp->queue = queue; + + mpp_dbg_core("%s attach queue as core %d\n", + dev_name(mpp->dev), mpp->core_id); + if (queue->task_capacity > mpp->task_capacity) queue->task_capacity = mpp->task_capacity; + +done: mutex_unlock(&queue->dev_lock); } @@ -896,7 +920,15 @@ static void mpp_detach_workqueue(struct mpp_dev *mpp) if (queue) { mutex_lock(&queue->dev_lock); + + queue->cores[mpp->core_id] = NULL; + queue->core_count--; + + clear_bit(queue->core_count, &queue->core_idle); list_del_init(&mpp->queue_link); + + mpp->queue = NULL; + mutex_unlock(&queue->dev_lock); } } @@ -1571,7 +1603,7 @@ int mpp_task_init(struct mpp_session *session, int mpp_task_finish(struct mpp_session *session, struct mpp_task *task) { - struct mpp_dev *mpp = session->mpp; + struct mpp_dev *mpp = task->mpp ? task->mpp : session->mpp; if (mpp->dev_ops->finish) mpp->dev_ops->finish(mpp, task); @@ -1722,6 +1754,16 @@ int mpp_dev_probe(struct mpp_dev *mpp, /* Get disable auto frequent flag from dtsi */ mpp->auto_freq_en = !device_property_read_bool(dev, "rockchip,disable-auto-freq"); + /* read link table capacity */ + ret = of_property_read_u32(np, "rockchip,task-capacity", + &mpp->task_capacity); + if (ret) + mpp->task_capacity = 1; + + mpp->dev = dev; + mpp->hw_ops = mpp->var->hw_ops; + mpp->dev_ops = mpp->var->dev_ops; + /* Get and attach to service */ ret = mpp_attach_service(mpp, dev); if (ret) { @@ -1729,21 +1771,12 @@ int mpp_dev_probe(struct mpp_dev *mpp, return -ENODEV; } - mpp->dev = dev; - mpp->hw_ops = mpp->var->hw_ops; - mpp->dev_ops = mpp->var->dev_ops; - - /* read link table capacity */ - ret = of_property_read_u32(np, "rockchip,task-capacity", - &mpp->task_capacity); - if (ret) { - mpp->task_capacity = 1; - + if (mpp->task_capacity == 1) { /* power domain autosuspend delay 2s */ pm_runtime_set_autosuspend_delay(dev, 2000); pm_runtime_use_autosuspend(dev); } else { - dev_info(dev, "%d task capacity link mode detected\n", + dev_info(dev, "link mode task capacity %d\n", mpp->task_capacity); /* do not setup autosuspend on multi task device */ } @@ -1945,7 +1978,7 @@ int mpp_time_record(struct mpp_task *task) int mpp_time_diff(struct mpp_task *task) { struct timespec64 end; - struct mpp_dev *mpp = task->session->mpp; + struct mpp_dev *mpp = task->mpp ? task->mpp : task->session->mpp; ktime_get_real_ts64(&end); mpp_debug(DEBUG_TIMING, "%s: pid: %d, session: %p, time: %lld us\n", diff --git a/drivers/video/rockchip/mpp/mpp_common.h b/drivers/video/rockchip/mpp/mpp_common.h index 4b50311aa65d..bf16a12dd395 100644 --- a/drivers/video/rockchip/mpp/mpp_common.h +++ b/drivers/video/rockchip/mpp/mpp_common.h @@ -39,6 +39,9 @@ /* grf mask for get value */ #define MPP_GRF_VAL_MASK (0xFFFF) +/* max 4 cores supported */ +#define MPP_MAX_CORE_NUM (4) + /** * Device type: classified by hardware feature */ @@ -318,7 +321,9 @@ struct mpp_dev { struct platform_device *pdev_srv; struct mpp_service *srv; + /* multi-core data */ struct list_head queue_link; + s32 core_id; }; struct mpp_task; @@ -411,6 +416,10 @@ struct mpp_task { u32 *reg; /* event for session wait thread */ wait_queue_head_t wait; + + /* for multi-core */ + struct mpp_dev *mpp; + s32 core_id; }; struct mpp_taskqueue { @@ -447,6 +456,11 @@ struct mpp_taskqueue { * device task capacity which is attached to the taskqueue */ u32 task_capacity; + + /* multi-core task distribution */ + struct mpp_dev *cores[MPP_MAX_CORE_NUM]; + unsigned long core_idle; + u32 core_count; }; struct mpp_reset_group { diff --git a/drivers/video/rockchip/mpp/mpp_debug.h b/drivers/video/rockchip/mpp/mpp_debug.h index 9e42adfe1571..5fe843a479ee 100644 --- a/drivers/video/rockchip/mpp/mpp_debug.h +++ b/drivers/video/rockchip/mpp/mpp_debug.h @@ -50,6 +50,9 @@ #define DEBUG_SESSION 0x00400000 #define DEBUG_DEVICE 0x00800000 +#define DEBUG_CCU 0x01000000 +#define DEBUG_CORE 0x02000000 + #define PRINT_FUNCTION 0x80000000 #define PRINT_LINE 0x40000000 @@ -106,4 +109,19 @@ extern unsigned int mpp_dev_debug; } \ } while (0) +#define mpp_dbg_ccu(fmt, args...) \ + do { \ + if (unlikely(mpp_dev_debug & DEBUG_CCU)) { \ + pr_info("%s:%d: " fmt, \ + __func__, __LINE__, ##args); \ + } \ + } while (0) + +#define mpp_dbg_core(fmt, args...) \ + do { \ + if (unlikely(mpp_dev_debug & DEBUG_CORE)) { \ + pr_info(fmt, ##args); \ + } \ + } while (0) + #endif diff --git a/drivers/video/rockchip/mpp/mpp_rkvenc2.c b/drivers/video/rockchip/mpp/mpp_rkvenc2.c index cd42b040e32e..56d3695ba658 100644 --- a/drivers/video/rockchip/mpp/mpp_rkvenc2.c +++ b/drivers/video/rockchip/mpp/mpp_rkvenc2.c @@ -723,6 +723,32 @@ static void *rkvenc_ccu_alloc_task(struct mpp_session *session, return rkvenc_alloc_task(session, msgs); } +static void *rkvenc2_prepare(struct mpp_dev *mpp, struct mpp_task *mpp_task) +{ + struct mpp_taskqueue *queue = mpp->queue; + unsigned long flags; + s32 core_id; + + spin_lock_irqsave(&queue->running_lock, flags); + + core_id = find_first_bit(&queue->core_idle, queue->core_count); + + if (core_id >= queue->core_count) { + mpp_task = NULL; + mpp_dbg_core("core %d all busy %lx\n", core_id, queue->core_idle); + } else { + mpp_dbg_core("core %d set idle %lx\n", core_id, queue->core_idle); + + clear_bit(core_id, &queue->core_idle); + mpp_task->mpp = queue->cores[core_id]; + mpp_task->core_id = core_id; + } + + spin_unlock_irqrestore(&queue->running_lock, flags); + + return mpp_task; +} + static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task) { u32 i, j; @@ -760,8 +786,16 @@ static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task) } } + if (mpp_debug_unlikely(DEBUG_CORE)) + dev_info(mpp->dev, "reg[%03x] %08x\n", 0x304, + mpp_read_relaxed(mpp, 0x304)); + + /* flush tlb before starting hardware */ + mpp_iommu_flush_tlb(mpp->iommu_info); + /* init current task */ mpp->cur_task = mpp_task; + /* Flush the register before the start the device */ wmb(); mpp_write(mpp, enc->hw_info->enc_start_base, start_val); @@ -781,6 +815,7 @@ static int rkvenc_irq(struct mpp_dev *mpp) mpp->irq_status = mpp_read(mpp, hw->int_sta_base); if (!mpp->irq_status) return IRQ_NONE; + mpp_write(mpp, hw->int_mask_base, 0x100); mpp_write(mpp, hw->int_clr_base, 0xffffffff); udelay(5); @@ -796,6 +831,7 @@ static int rkvenc_isr(struct mpp_dev *mpp) struct rkvenc_task *task; struct mpp_task *mpp_task; struct rkvenc_dev *enc = to_rkvenc_dev(mpp); + struct mpp_taskqueue *queue = mpp->queue; mpp_debug_enter(); @@ -808,9 +844,15 @@ static int rkvenc_isr(struct mpp_dev *mpp) mpp_task = mpp->cur_task; mpp_time_diff(mpp_task); mpp->cur_task = NULL; + + if (mpp_task->mpp && mpp_task->mpp != mpp) + dev_err(mpp->dev, "mismatch core dev %p:%p\n", mpp_task->mpp, mpp); + task = to_rkvenc_task(mpp_task); task->irq_status = mpp->irq_status; - mpp_debug(DEBUG_IRQ_STATUS, "irq_status: %08x\n", task->irq_status); + + mpp_debug(DEBUG_IRQ_STATUS, "%s irq_status: %08x\n", + dev_name(mpp->dev), task->irq_status); if (task->irq_status & enc->hw_info->err_mask) { atomic_inc(&mpp->reset_request); @@ -820,6 +862,9 @@ static int rkvenc_isr(struct mpp_dev *mpp) } mpp_task_finish(mpp_task->session, mpp_task); + set_bit(mpp->core_id, &queue->core_idle); + mpp_dbg_core("core %d isr idle %lx\n", mpp->core_id, queue->core_idle); + mpp_debug_leave(); return IRQ_HANDLED; @@ -1049,8 +1094,16 @@ static int rkvenc_show_session_info(struct seq_file *seq, void *offset) static int rkvenc_procfs_init(struct mpp_dev *mpp) { struct rkvenc_dev *enc = to_rkvenc_dev(mpp); + char name[32]; - enc->procfs = proc_mkdir(mpp->dev->of_node->name, mpp->srv->procfs); + if (!mpp->dev || !mpp->dev->of_node || !mpp->dev->of_node->name || + !mpp->srv || !mpp->srv->procfs) + return -EINVAL; + + snprintf(name, sizeof(name) - 1, "%s%d", + mpp->dev->of_node->name, mpp->core_id); + + enc->procfs = proc_mkdir(name, mpp->srv->procfs); if (IS_ERR_OR_NULL(enc->procfs)) { mpp_err("failed on open procfs\n"); enc->procfs = NULL; @@ -1142,6 +1195,7 @@ static int rkvenc_reset(struct mpp_dev *mpp) { struct rkvenc_dev *enc = to_rkvenc_dev(mpp); struct rkvenc_hw_info *hw = enc->hw_info; + struct mpp_taskqueue *queue = mpp->queue; mpp_debug_enter(); @@ -1165,6 +1219,9 @@ static int rkvenc_reset(struct mpp_dev *mpp) rockchip_pmu_idle_request(mpp->dev, false); } + set_bit(mpp->core_id, &queue->core_idle); + mpp_dbg_core("core %d reset idle %lx\n", mpp->core_id, queue->core_idle); + mpp_debug_leave(); return 0; @@ -1227,6 +1284,7 @@ static struct mpp_dev_ops rkvenc_dev_ops_v2 = { static struct mpp_dev_ops rkvenc_ccu_dev_ops = { .alloc_task = rkvenc_ccu_alloc_task, + .prepare = rkvenc2_prepare, .run = rkvenc_run, .irq = rkvenc_irq, .isr = rkvenc_isr, @@ -1332,11 +1390,12 @@ static int rkvenc_attach_ccu(struct device *dev, struct rkvenc_dev *enc) cur_info = enc->mpp.iommu_info; cur_info->domain = ccu_info->domain; + cur_info->rw_sem = ccu_info->rw_sem; mpp_iommu_attach(cur_info); } enc->ccu = ccu; - dev_info(dev, "attach ccu success\n"); + dev_info(dev, "attach ccu as core %d\n", enc->mpp.core_id); mpp_debug_enter(); return 0; @@ -1356,10 +1415,9 @@ static int rkvenc2_alloc_rcbbuf(struct platform_device *pdev, struct rkvenc_dev /* get rcb iova start and size */ ret = device_property_read_u32_array(dev, "rockchip,rcb-iova", vals, 2); - if (ret) { - dev_err(dev, "could not find property rcb-iova\n"); + if (ret) return ret; - } + iova = PAGE_ALIGN(vals[0]); sram_used = PAGE_ALIGN(vals[1]); if (!sram_used) { @@ -1447,8 +1505,6 @@ static int rkvenc_core_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct rkvenc_dev *enc = NULL; struct mpp_dev *mpp = NULL; - const struct of_device_id *match = NULL; - enc = devm_kzalloc(dev, sizeof(*enc), GFP_KERNEL); if (!enc) @@ -1458,9 +1514,14 @@ static int rkvenc_core_probe(struct platform_device *pdev) platform_set_drvdata(pdev, enc); if (pdev->dev.of_node) { - match = of_match_node(mpp_rkvenc_dt_match, pdev->dev.of_node); + struct device_node *np = pdev->dev.of_node; + const struct of_device_id *match = NULL; + + match = of_match_node(mpp_rkvenc_dt_match, np); if (match) mpp->var = (struct mpp_dev_var *)match->data; + + mpp->core_id = of_alias_get_id(np, "rkvenc"); } ret = mpp_dev_probe(mpp, pdev); @@ -1625,7 +1686,9 @@ static void rkvenc_shutdown(struct platform_device *pdev) dev_info(dev, "shutdown device\n"); - atomic_inc(&mpp->srv->shutdown_request); + if (mpp->srv) + atomic_inc(&mpp->srv->shutdown_request); + ret = readx_poll_timeout(atomic_read, &mpp->task_count, val, val == 0, 1000, 200000);