diff --git a/drivers/video/rockchip/rga3/include/rga_drv.h b/drivers/video/rockchip/rga3/include/rga_drv.h
index 23467d74ed3f..64473988350c 100644
--- a/drivers/video/rockchip/rga3/include/rga_drv.h
+++ b/drivers/video/rockchip/rga3/include/rga_drv.h
@@ -217,7 +217,7 @@ struct rga_scheduler_t {
 	struct clk *clks[RGA_MAX_BUS_CLK];
 	int num_clks;
 
-	struct kref pd_refcount;
+	int pd_refcount;
 
 	struct rga_job *running_job;
 	struct list_head todo_list;
@@ -278,6 +278,5 @@ static inline void rga_write(int value, int offset, struct rga_scheduler_t *rga_
 
 int rga_power_enable(struct rga_scheduler_t *rga_scheduler);
 int rga_power_disable(struct rga_scheduler_t *rga_scheduler);
-void rga_kref_disable_power(struct kref *ref);
 
 #endif /* __LINUX_RGA_FENCE_H_ */
diff --git a/drivers/video/rockchip/rga3/rga_debugger.c b/drivers/video/rockchip/rga3/rga_debugger.c
index 1f1eea2a2865..1b20a695e6f1 100644
--- a/drivers/video/rockchip/rga3/rga_debugger.c
+++ b/drivers/video/rockchip/rga3/rga_debugger.c
@@ -156,6 +156,7 @@ static int rga_scheduler_show(struct seq_file *m, void *data)
 		seq_printf(m, "scheduler[%d]: core = %d\n",
 			i, rga_scheduler->core);
 		seq_printf(m, "-----------------------------------\n");
+		seq_printf(m, "pd_ref = %d\n", rga_scheduler->pd_refcount);
 	}
 
 	return 0;
diff --git a/drivers/video/rockchip/rga3/rga_drv.c b/drivers/video/rockchip/rga3/rga_drv.c
index 8cc85c397853..33cbb6327174 100644
--- a/drivers/video/rockchip/rga3/rga_drv.c
+++ b/drivers/video/rockchip/rga3/rga_drv.c
@@ -77,6 +77,8 @@ err_enable_clk:
 	pm_relax(rga_scheduler->dev);
 	pm_runtime_put_sync_suspend(rga_scheduler->dev);
 
+	rga_scheduler->pd_refcount++;
+
 	return ret;
 }
 
@@ -91,18 +93,11 @@ int rga_power_disable(struct rga_scheduler_t *rga_scheduler)
 	pm_relax(rga_scheduler->dev);
 	pm_runtime_put_sync_suspend(rga_scheduler->dev);
 
+	rga_scheduler->pd_refcount--;
+
 	return 0;
 }
 
-void rga_kref_disable_power(struct kref *ref)
-{
-	struct rga_scheduler_t *rga_scheduler = NULL;
-
-	rga_scheduler = container_of(ref, struct rga_scheduler_t, pd_refcount);
-
-	rga_power_disable(rga_scheduler);
-}
-
 #endif //CONFIG_ROCKCHIP_FPGA
 
 static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg)
diff --git a/drivers/video/rockchip/rga3/rga_job.c b/drivers/video/rockchip/rga3/rga_job.c
index d2794c619b97..7dfb6733c8b2 100644
--- a/drivers/video/rockchip/rga3/rga_job.c
+++ b/drivers/video/rockchip/rga3/rga_job.c
@@ -110,6 +110,13 @@ static int rga_job_run(struct rga_job *job, struct rga_scheduler_t *scheduler)
 {
 	int ret = 0;
 
+	/* enable power */
+	ret = rga_power_enable(scheduler);
+	if (ret < 0) {
+		pr_err("power enable failed");
+		return ret;
+	}
+
 	ret = rga_dma_get_info(job);
 	if (ret < 0) {
 		pr_err("dma buf get failed");
@@ -132,7 +139,11 @@ static int rga_job_run(struct rga_job *job, struct rga_scheduler_t *scheduler)
 	if (RGA_DEBUG_MSG)
 		print_job_info(job);
 
+	return ret;
+
 failed:
+	rga_power_disable(scheduler);
+
 	return ret;
 }
 
@@ -197,8 +208,10 @@ void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret)
 	ktime_t now;
 
 	spin_lock_irqsave(&rga_scheduler->irq_lock, flags);
+
 	job = rga_scheduler->running_job;
 	rga_scheduler->running_job = NULL;
+
 	spin_unlock_irqrestore(&rga_scheduler->irq_lock, flags);
 
 	job->flags |= RGA_JOB_DONE;
@@ -216,8 +229,6 @@ void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret)
 
 	rga_dma_put_info(job);
 
-	kref_put(&rga_scheduler->pd_refcount, rga_kref_disable_power);
-
 	mmdrop(job->mm);
 
 	if (job->out_fence)
@@ -229,6 +240,8 @@ void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret)
 		rga_job_cleanup(job);
 
 	rga_job_next(rga_scheduler);
+
+	rga_power_disable(rga_scheduler);
 }
 
 static int rga_set_feature(struct rga_req *rga_base)
@@ -540,8 +553,6 @@ static struct rga_scheduler_t *rga_job_schedule(struct rga_job *job)
 	struct rga_scheduler_t *scheduler = NULL;
 	struct rga_job *job_pos;
 	bool first_match = 0;
-	bool first_open_pd = false;
-	int ret = 0;
 
 	if (rga_drvdata->num_of_scheduler > 1) {
 		job->core = rga_job_assign(job);
@@ -561,13 +572,6 @@ static struct rga_scheduler_t *rga_job_schedule(struct rga_job *job)
 
 	spin_lock_irqsave(&scheduler->irq_lock, flags);
 
-	if (list_empty(&scheduler->todo_list) && !scheduler->running_job) {
-		kref_init(&scheduler->pd_refcount);
-		first_open_pd = true;
-	} else {
-		kref_get(&scheduler->pd_refcount);
-	}
-
 	/* priority policy set by userspace */
 	if (list_empty(&scheduler->todo_list)
 		|| (job->priority == RGA_SCHED_PRIORITY_DEFAULT)) {
@@ -596,15 +600,6 @@ static struct rga_scheduler_t *rga_job_schedule(struct rga_job *job)
 
 	spin_unlock_irqrestore(&scheduler->irq_lock, flags);
 
-	/* enable power */
-	if (first_open_pd) {
-		ret = rga_power_enable(scheduler);
-		if (ret < 0) {
-			pr_err("power enable failed");
-			return NULL;
-		}
-	}
-
 	rga_job_next(scheduler);
 
 	return scheduler;