video: rockchip: mpp: rkvenc2: Fix dual core issue

The rkvenc2 dual core mode will split one large frame into two tile as
task 0 and 1. The task 0 must be started before task 1 otherwise
hardware will generate error.

So we need some changes:

1. Bind two core into one taskqueue. If each core is binded to one
thread then the real hardware config and start timing is not certain.
NOTE: current mode still has case that one extra task is insert between
task 0 and 1.

2. Record core info and core id when multi-core is attached to the same
taskqueue.

3. Add prepare stage to decide which core is used to run the task and
set the struce mpp_dev *mpp in struct mpp_task.

4. Correspondingly the free task should use the mpp_task->mpp if it is a
multi-core mode task.

5. Remove mpp_iommu_attach in mpp_task_run. The mpp_iommu_attach is for
switching different devices which has shared resource like vpu and
rkhevc in rk3568 and rk312x. These chipsets are not supported on 5.10
kernel.

Signed-off-by: Herman Chen <herman.chen@rock-chips.com>
Change-Id: Ide52d59cca3adec44e05ab4bfc5bac7d713ad628

arch/arm64/boot/dts/rockchip/rk3588-evb.dtsi

@@ -923,6 +923,10 @@
 	status = "okay";
 };
 
+&rkvenc_ccu {
+	status = "okay";
+};
+
 &rkvenc0 {
 	status = "okay";
 };
@@ -931,6 +935,14 @@
 	status = "okay";
 };
 
+&rkvenc1 {
+	status = "okay";
+};
+
+&rkvenc1_mmu {
+	status = "okay";
+};
+
 &saradc {
 	status = "okay";
 	vref-supply = <&vcc_1v8_s0>;

arch/arm64/boot/dts/rockchip/rk3588-nvr.dtsi

@@ -167,6 +167,10 @@
 	status = "okay";
 };
 
+&rkvenc_ccu {
+	status = "okay";
+};
+
 &rkvenc0 {
 	status = "okay";
 };
@@ -175,6 +179,14 @@
 	status = "okay";
 };
 
+&rkvenc1 {
+	status = "okay";
+};
+
+&rkvenc1_mmu {
+	status = "okay";
+};
+
 &saradc {
 	status = "okay";
 	vref-supply = <&vcc_1v8_cam_s0>;

arch/arm64/boot/dts/rockchip/rk3588s-evb.dtsi

@@ -922,6 +922,10 @@
 	status = "okay";
 };
 
+&rkvenc_ccu {
+	status = "okay";
+};
+
 &rkvenc0 {
 	status = "okay";
 };
@@ -930,6 +934,14 @@
 	status = "okay";
 };
 
+&rkvenc1 {
+	status = "okay";
+};
+
+&rkvenc1_mmu {
+	status = "okay";
+};
+
 &saradc {
 	status = "okay";
 	vref-supply = <&vcc_1v8_s0>;

arch/arm64/boot/dts/rockchip/rk3588s.dtsi

@@ -39,6 +39,8 @@
 		rkcif_mipi_lvds1= &rkcif_mipi_lvds1;
 		rkcif_mipi_lvds2= &rkcif_mipi_lvds2;
 		rkcif_mipi_lvds3= &rkcif_mipi_lvds3;
+		rkvenc0 = &rkvenc0;
+		rkvenc1 = &rkvenc1;
 		serial0 = &uart0;
 		serial1 = &uart1;
 		serial2 = &uart2;
@@ -2026,8 +2028,8 @@
 		status = "disabled";
 	};
 
-	rkvenc0: rkvenc@fdbd0000 {
-		compatible = "rockchip,rkv-encoder-v2";
+	rkvenc0: rkvenc-core@fdbd0000 {
+		compatible = "rockchip,rkv-encoder-v2-core";
 		reg = <0x0 0xfdbd0000 0x0 0x6000>;
 		interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-names = "irq_rkvenc0";
@@ -2063,7 +2065,7 @@
 		status = "disabled";
 	};
 
-	rkvenc_core1: rkvenc@fdbe0000 {
+	rkvenc1: rkvenc-core@fdbe0000 {
 		compatible = "rockchip,rkv-encoder-v2-core";
 		reg = <0x0 0xfdbe0000 0x0 0x6000>;
 		interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
@@ -2078,7 +2080,7 @@
 		iommus = <&rkvenc1_mmu>;
 		rockchip,srv = <&mpp_srv>;
 		rockchip,ccu = <&rkvenc_ccu>;
-		rockchip,taskqueue-node = <8>;
+		rockchip,taskqueue-node = <7>;
 		rockchip,task-capacity = <8>;
 		power-domains = <&power RK3588_PD_VENC1>;
 		status = "disabled";

drivers/video/rockchip/mpp/mpp_common.c

@@ -398,18 +398,20 @@ static void mpp_free_task(struct kref *ref)
 	}
 	session = task->session;
 
-	mpp_debug_func(DEBUG_TASK_INFO,
-		       "session=%p, task=%p, state=0x%lx, abort_request=%d\n",
-		       session, task, task->state,
+	mpp_debug_func(DEBUG_TASK_INFO, "task %d:%d state 0x%lx abort %d\n",
+		       session->index, task->task_index, task->state,
 		       atomic_read(&task->abort_request));
-	if (!session->mpp) {
-		mpp_err("session %p, session->mpp is null.\n", session);
+
+	mpp = task->mpp ? task->mpp : session->mpp;
+	if (!mpp) {
+		mpp_err("task %d:%d mpp is null.\n",
+			session->index, task->task_index);
 		return;
 	}
-	mpp = session->mpp;
 
 	if (mpp->dev_ops->free_task)
 		mpp->dev_ops->free_task(session, task);
+
 	/* Decrease reference count */
 	atomic_dec(&session->task_count);
 	atomic_dec(&mpp->task_count);
@@ -581,7 +583,6 @@ int mpp_dev_reset(struct mpp_dev *mpp)
 	mpp_iommu_down_write(mpp->iommu_info);
 	mpp_reset_down_write(mpp->reset_group);
 	atomic_set(&mpp->reset_request, 0);
-	mpp_iommu_detach(mpp->iommu_info);
 
 	rockchip_save_qos(mpp->dev);
 	if (mpp->hw_ops->reset)
@@ -594,7 +595,6 @@ int mpp_dev_reset(struct mpp_dev *mpp)
 	 */
 	mpp_iommu_refresh(mpp->iommu_info, mpp->dev);
 
-	mpp_iommu_attach(mpp->iommu_info);
 	mpp_reset_up_write(mpp->reset_group);
 	mpp_iommu_up_write(mpp->iommu_info);
 
@@ -623,15 +623,6 @@ static int mpp_task_run(struct mpp_dev *mpp,
 	} else {
 		mpp_set_grf(mpp->grf_info);
 	}
-	/*
-	 * for iommu share hardware, should attach to ensure
-	 * working in current device
-	 */
-	ret = mpp_iommu_attach(mpp->iommu_info);
-	if (ret) {
-		dev_err(mpp->dev, "mpp_iommu_attach failed\n");
-		return -ENODATA;
-	}
 
 	mpp_power_on(mpp);
 	mpp_time_record(task);
@@ -697,10 +688,12 @@ static void mpp_task_worker_default(struct kthread_work *work_s)
 	 */
 	/* Push a pending task to running queue */
 	if (task) {
+		struct mpp_dev *task_mpp = task->mpp ? task->mpp : mpp;
+
 		mpp_taskqueue_pending_to_run(queue, task);
 		set_bit(TASK_STATE_RUNNING, &task->state);
-		if (mpp_task_run(mpp, task))
-			mpp_taskqueue_pop_running(mpp->queue, task);
+		if (mpp_task_run(task_mpp, task))
+			mpp_taskqueue_pop_running(queue, task);
 	}
 
 done:
@@ -881,12 +874,43 @@ struct mpp_taskqueue *mpp_taskqueue_init(struct device *dev)
 static void mpp_attach_workqueue(struct mpp_dev *mpp,
 				 struct mpp_taskqueue *queue)
 {
-	mpp->queue = queue;
+	s32 core_id;
+
 	INIT_LIST_HEAD(&mpp->queue_link);
+
 	mutex_lock(&queue->dev_lock);
+
+	if (mpp->core_id >= 0)
+		core_id = mpp->core_id;
+	else
+		core_id = queue->core_count;
+
+	if (core_id < 0 || core_id >= MPP_MAX_CORE_NUM) {
+		dev_err(mpp->dev, "invalid core id %d\n", core_id);
+		goto done;
+	}
+
+	if (queue->cores[core_id]) {
+		dev_err(mpp->dev, "can not attach device with same id %d", core_id);
+		goto done;
+	}
+
+	queue->cores[core_id] = mpp;
+	queue->core_count++;
+
+	set_bit(core_id, &queue->core_idle);
 	list_add_tail(&mpp->queue_link, &queue->dev_list);
+
+	mpp->core_id = core_id;
+	mpp->queue = queue;
+
+	mpp_dbg_core("%s attach queue as core %d\n",
+			dev_name(mpp->dev), mpp->core_id);
+
 	if (queue->task_capacity > mpp->task_capacity)
 		queue->task_capacity = mpp->task_capacity;
+
+done:
 	mutex_unlock(&queue->dev_lock);
 }
 
@@ -896,7 +920,15 @@ static void mpp_detach_workqueue(struct mpp_dev *mpp)
 
 	if (queue) {
 		mutex_lock(&queue->dev_lock);
+
+		queue->cores[mpp->core_id] = NULL;
+		queue->core_count--;
+
+		clear_bit(queue->core_count, &queue->core_idle);
 		list_del_init(&mpp->queue_link);
+
+		mpp->queue = NULL;
+
 		mutex_unlock(&queue->dev_lock);
 	}
 }
@@ -1571,7 +1603,7 @@ int mpp_task_init(struct mpp_session *session,
 int mpp_task_finish(struct mpp_session *session,
 		    struct mpp_task *task)
 {
-	struct mpp_dev *mpp = session->mpp;
+	struct mpp_dev *mpp = task->mpp ? task->mpp : session->mpp;
 
 	if (mpp->dev_ops->finish)
 		mpp->dev_ops->finish(mpp, task);
@@ -1722,6 +1754,16 @@ int mpp_dev_probe(struct mpp_dev *mpp,
 	/* Get disable auto frequent flag from dtsi */
 	mpp->auto_freq_en = !device_property_read_bool(dev, "rockchip,disable-auto-freq");
 
+	/* read link table capacity */
+	ret = of_property_read_u32(np, "rockchip,task-capacity",
+				   &mpp->task_capacity);
+	if (ret)
+		mpp->task_capacity = 1;
+
+	mpp->dev = dev;
+	mpp->hw_ops = mpp->var->hw_ops;
+	mpp->dev_ops = mpp->var->dev_ops;
+
 	/* Get and attach to service */
 	ret = mpp_attach_service(mpp, dev);
 	if (ret) {
@@ -1729,21 +1771,12 @@ int mpp_dev_probe(struct mpp_dev *mpp,
 		return -ENODEV;
 	}
 
-	mpp->dev = dev;
-	mpp->hw_ops = mpp->var->hw_ops;
-	mpp->dev_ops = mpp->var->dev_ops;
-
-	/* read link table capacity */
-	ret = of_property_read_u32(np, "rockchip,task-capacity",
-				   &mpp->task_capacity);
-	if (ret) {
-		mpp->task_capacity = 1;
-
+	if (mpp->task_capacity == 1) {
 		/* power domain autosuspend delay 2s */
 		pm_runtime_set_autosuspend_delay(dev, 2000);
 		pm_runtime_use_autosuspend(dev);
 	} else {
-		dev_info(dev, "%d task capacity link mode detected\n",
+		dev_info(dev, "link mode task capacity %d\n",
 			 mpp->task_capacity);
 		/* do not setup autosuspend on multi task device */
 	}
@@ -1945,7 +1978,7 @@ int mpp_time_record(struct mpp_task *task)
 int mpp_time_diff(struct mpp_task *task)
 {
 	struct timespec64 end;
-	struct mpp_dev *mpp = task->session->mpp;
+	struct mpp_dev *mpp = task->mpp ? task->mpp : task->session->mpp;
 
 	ktime_get_real_ts64(&end);
 	mpp_debug(DEBUG_TIMING, "%s: pid: %d, session: %p, time: %lld us\n",

drivers/video/rockchip/mpp/mpp_common.h

@@ -39,6 +39,9 @@
 /* grf mask for get value */
 #define MPP_GRF_VAL_MASK		(0xFFFF)
 
+/* max 4 cores supported */
+#define MPP_MAX_CORE_NUM		(4)
+
 /**
  * Device type: classified by hardware feature
  */
@@ -318,7 +321,9 @@ struct mpp_dev {
 	struct platform_device *pdev_srv;
 	struct mpp_service *srv;
 
+	/* multi-core data */
 	struct list_head queue_link;
+	s32 core_id;
 };
 
 struct mpp_task;
@@ -411,6 +416,10 @@ struct mpp_task {
 	u32 *reg;
 	/* event for session wait thread */
 	wait_queue_head_t wait;
+
+	/* for multi-core */
+	struct mpp_dev *mpp;
+	s32 core_id;
 };
 
 struct mpp_taskqueue {
@@ -447,6 +456,11 @@ struct mpp_taskqueue {
 	 * device task capacity which is attached to the taskqueue
 	 */
 	u32 task_capacity;
+
+	/* multi-core task distribution */
+	struct mpp_dev *cores[MPP_MAX_CORE_NUM];
+	unsigned long core_idle;
+	u32 core_count;
 };
 
 struct mpp_reset_group {

drivers/video/rockchip/mpp/mpp_debug.h

@@ -50,6 +50,9 @@
 #define DEBUG_SESSION				0x00400000
 #define DEBUG_DEVICE				0x00800000
 
+#define DEBUG_CCU				0x01000000
+#define DEBUG_CORE				0x02000000
+
 #define PRINT_FUNCTION				0x80000000
 #define PRINT_LINE				0x40000000
 
@@ -106,4 +109,19 @@ extern unsigned int mpp_dev_debug;
 		}						\
 	} while (0)
 
+#define mpp_dbg_ccu(fmt, args...)				\
+	do {							\
+		if (unlikely(mpp_dev_debug & DEBUG_CCU)) {	\
+			pr_info("%s:%d: " fmt,			\
+				 __func__, __LINE__, ##args);	\
+		}						\
+	} while (0)
+
+#define mpp_dbg_core(fmt, args...)				\
+	do {							\
+		if (unlikely(mpp_dev_debug & DEBUG_CORE)) {	\
+			pr_info(fmt, ##args);			\
+		}						\
+	} while (0)
+
 #endif

drivers/video/rockchip/mpp/mpp_rkvenc2.c

@@ -723,6 +723,32 @@ static void *rkvenc_ccu_alloc_task(struct mpp_session *session,
 	return rkvenc_alloc_task(session, msgs);
 }
 
+static void *rkvenc2_prepare(struct mpp_dev *mpp, struct mpp_task *mpp_task)
+{
+	struct mpp_taskqueue *queue = mpp->queue;
+	unsigned long flags;
+	s32 core_id;
+
+	spin_lock_irqsave(&queue->running_lock, flags);
+
+	core_id = find_first_bit(&queue->core_idle, queue->core_count);
+
+	if (core_id >= queue->core_count) {
+		mpp_task = NULL;
+		mpp_dbg_core("core %d all busy %lx\n", core_id, queue->core_idle);
+	} else {
+		mpp_dbg_core("core %d set idle %lx\n", core_id, queue->core_idle);
+
+		clear_bit(core_id, &queue->core_idle);
+		mpp_task->mpp = queue->cores[core_id];
+		mpp_task->core_id = core_id;
+	}
+
+	spin_unlock_irqrestore(&queue->running_lock, flags);
+
+	return mpp_task;
+}
+
 static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task)
 {
 	u32 i, j;
@@ -760,8 +786,16 @@ static int rkvenc_run(struct mpp_dev *mpp, struct mpp_task *mpp_task)
 		}
 	}
 
+	if (mpp_debug_unlikely(DEBUG_CORE))
+		dev_info(mpp->dev, "reg[%03x] %08x\n", 0x304,
+			 mpp_read_relaxed(mpp, 0x304));
+
+	/* flush tlb before starting hardware */
+	mpp_iommu_flush_tlb(mpp->iommu_info);
+
 	/* init current task */
 	mpp->cur_task = mpp_task;
+
 	/* Flush the register before the start the device */
 	wmb();
 	mpp_write(mpp, enc->hw_info->enc_start_base, start_val);
@@ -781,6 +815,7 @@ static int rkvenc_irq(struct mpp_dev *mpp)
 	mpp->irq_status = mpp_read(mpp, hw->int_sta_base);
 	if (!mpp->irq_status)
 		return IRQ_NONE;
+
 	mpp_write(mpp, hw->int_mask_base, 0x100);
 	mpp_write(mpp, hw->int_clr_base, 0xffffffff);
 	udelay(5);
@@ -796,6 +831,7 @@ static int rkvenc_isr(struct mpp_dev *mpp)
 	struct rkvenc_task *task;
 	struct mpp_task *mpp_task;
 	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
+	struct mpp_taskqueue *queue = mpp->queue;
 
 	mpp_debug_enter();
 
@@ -808,9 +844,15 @@ static int rkvenc_isr(struct mpp_dev *mpp)
 	mpp_task = mpp->cur_task;
 	mpp_time_diff(mpp_task);
 	mpp->cur_task = NULL;
+
+	if (mpp_task->mpp && mpp_task->mpp != mpp)
+		dev_err(mpp->dev, "mismatch core dev %p:%p\n", mpp_task->mpp, mpp);
+
 	task = to_rkvenc_task(mpp_task);
 	task->irq_status = mpp->irq_status;
-	mpp_debug(DEBUG_IRQ_STATUS, "irq_status: %08x\n", task->irq_status);
+
+	mpp_debug(DEBUG_IRQ_STATUS, "%s irq_status: %08x\n",
+		  dev_name(mpp->dev), task->irq_status);
 
 	if (task->irq_status & enc->hw_info->err_mask) {
 		atomic_inc(&mpp->reset_request);
@@ -820,6 +862,9 @@ static int rkvenc_isr(struct mpp_dev *mpp)
 	}
 	mpp_task_finish(mpp_task->session, mpp_task);
 
+	set_bit(mpp->core_id, &queue->core_idle);
+	mpp_dbg_core("core %d isr idle %lx\n", mpp->core_id, queue->core_idle);
+
 	mpp_debug_leave();
 
 	return IRQ_HANDLED;
@@ -1049,8 +1094,16 @@ static int rkvenc_show_session_info(struct seq_file *seq, void *offset)
 static int rkvenc_procfs_init(struct mpp_dev *mpp)
 {
 	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
+	char name[32];
 
-	enc->procfs = proc_mkdir(mpp->dev->of_node->name, mpp->srv->procfs);
+	if (!mpp->dev || !mpp->dev->of_node || !mpp->dev->of_node->name ||
+	    !mpp->srv || !mpp->srv->procfs)
+		return -EINVAL;
+
+	snprintf(name, sizeof(name) - 1, "%s%d",
+		 mpp->dev->of_node->name, mpp->core_id);
+
+	enc->procfs = proc_mkdir(name, mpp->srv->procfs);
 	if (IS_ERR_OR_NULL(enc->procfs)) {
 		mpp_err("failed on open procfs\n");
 		enc->procfs = NULL;
@@ -1142,6 +1195,7 @@ static int rkvenc_reset(struct mpp_dev *mpp)
 {
 	struct rkvenc_dev *enc = to_rkvenc_dev(mpp);
 	struct rkvenc_hw_info *hw = enc->hw_info;
+	struct mpp_taskqueue *queue = mpp->queue;
 
 	mpp_debug_enter();
 
@@ -1165,6 +1219,9 @@ static int rkvenc_reset(struct mpp_dev *mpp)
 		rockchip_pmu_idle_request(mpp->dev, false);
 	}
 
+	set_bit(mpp->core_id, &queue->core_idle);
+	mpp_dbg_core("core %d reset idle %lx\n", mpp->core_id, queue->core_idle);
+
 	mpp_debug_leave();
 
 	return 0;
@@ -1227,6 +1284,7 @@ static struct mpp_dev_ops rkvenc_dev_ops_v2 = {
 
 static struct mpp_dev_ops rkvenc_ccu_dev_ops = {
 	.alloc_task = rkvenc_ccu_alloc_task,
+	.prepare = rkvenc2_prepare,
 	.run = rkvenc_run,
 	.irq = rkvenc_irq,
 	.isr = rkvenc_isr,
@@ -1332,11 +1390,12 @@ static int rkvenc_attach_ccu(struct device *dev, struct rkvenc_dev *enc)
 		cur_info = enc->mpp.iommu_info;
 
 		cur_info->domain = ccu_info->domain;
+		cur_info->rw_sem = ccu_info->rw_sem;
 		mpp_iommu_attach(cur_info);
 	}
 	enc->ccu = ccu;
 
-	dev_info(dev, "attach ccu success\n");
+	dev_info(dev, "attach ccu as core %d\n", enc->mpp.core_id);
 	mpp_debug_enter();
 
 	return 0;
@@ -1356,10 +1415,9 @@ static int rkvenc2_alloc_rcbbuf(struct platform_device *pdev, struct rkvenc_dev
 
 	/* get rcb iova start and size */
 	ret = device_property_read_u32_array(dev, "rockchip,rcb-iova", vals, 2);
-	if (ret) {
-		dev_err(dev, "could not find property rcb-iova\n");
+	if (ret)
 		return ret;
-	}
+
 	iova = PAGE_ALIGN(vals[0]);
 	sram_used = PAGE_ALIGN(vals[1]);
 	if (!sram_used) {
@@ -1447,8 +1505,6 @@ static int rkvenc_core_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct rkvenc_dev *enc = NULL;
 	struct mpp_dev *mpp = NULL;
-	const struct of_device_id *match = NULL;
-
 
 	enc = devm_kzalloc(dev, sizeof(*enc), GFP_KERNEL);
 	if (!enc)
@@ -1458,9 +1514,14 @@ static int rkvenc_core_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, enc);
 
 	if (pdev->dev.of_node) {
-		match = of_match_node(mpp_rkvenc_dt_match, pdev->dev.of_node);
+		struct device_node *np = pdev->dev.of_node;
+		const struct of_device_id *match = NULL;
+
+		match = of_match_node(mpp_rkvenc_dt_match, np);
 		if (match)
 			mpp->var = (struct mpp_dev_var *)match->data;
+
+		mpp->core_id = of_alias_get_id(np, "rkvenc");
 	}
 
 	ret = mpp_dev_probe(mpp, pdev);
@@ -1625,7 +1686,9 @@ static void rkvenc_shutdown(struct platform_device *pdev)
 
 	dev_info(dev, "shutdown device\n");
 
-	atomic_inc(&mpp->srv->shutdown_request);
+	if (mpp->srv)
+		atomic_inc(&mpp->srv->shutdown_request);
+
 	ret = readx_poll_timeout(atomic_read,
 				 &mpp->task_count,
 				 val, val == 0, 1000, 200000);