diff --git a/drivers/media/platform/rockchip/isp/capture_v21.c b/drivers/media/platform/rockchip/isp/capture_v21.c
index 650b8cb85659..2dc077a48d25 100644
--- a/drivers/media/platform/rockchip/isp/capture_v21.c
+++ b/drivers/media/platform/rockchip/isp/capture_v21.c
@@ -1331,6 +1331,7 @@ static void rkisp_stream_stop(struct rkisp_stream *stream)
 	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
 	unsigned long lock_flags = 0;
 	int ret = 0;
+	bool is_wait = true;
 
 	if (!dev->dmarx_dev.trigger &&
 	    (is_rdbk_stream(stream) || is_hdr_stream(stream))) {
@@ -1348,12 +1349,13 @@ static void rkisp_stream_stop(struct rkisp_stream *stream)
 		if (IS_HDR_RDBK(dev->rd_mode)) {
 			spin_lock_irqsave(&dev->hw_dev->rdbk_lock, lock_flags);
 			if (dev->hw_dev->cur_dev_id != dev->dev_id || dev->hw_dev->is_idle)
+				is_wait = false;
+			if (atomic_read(&dev->cap_dev.refcnt) == 1 && !is_wait)
 				dev->isp_state = ISP_STOP;
 			spin_unlock_irqrestore(&dev->hw_dev->rdbk_lock, lock_flags);
 		}
 	}
-	if (dev->isp_state & ISP_START &&
-	    !stream->ops->is_stream_stopped(stream)) {
+	if (is_wait && !stream->ops->is_stream_stopped(stream)) {
 		ret = wait_event_timeout(stream->done,
 					 !stream->streaming,
 					 msecs_to_jiffies(500));
diff --git a/drivers/media/platform/rockchip/isp/capture_v30.c b/drivers/media/platform/rockchip/isp/capture_v30.c
index 683df04e317a..d0ff8e295e0a 100644
--- a/drivers/media/platform/rockchip/isp/capture_v30.c
+++ b/drivers/media/platform/rockchip/isp/capture_v30.c
@@ -966,6 +966,7 @@ static void rkisp_stream_stop(struct rkisp_stream *stream)
 	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
 	unsigned long lock_flags = 0;
 	int ret = 0;
+	bool is_wait = true;
 
 	stream->stopping = true;
 	if (dev->hw_dev->is_single)
@@ -973,11 +974,12 @@ static void rkisp_stream_stop(struct rkisp_stream *stream)
 	if (IS_HDR_RDBK(dev->rd_mode)) {
 		spin_lock_irqsave(&dev->hw_dev->rdbk_lock, lock_flags);
 		if (dev->hw_dev->cur_dev_id != dev->dev_id || dev->hw_dev->is_idle)
+			is_wait = false;
+		if (atomic_read(&dev->cap_dev.refcnt) == 1 && !is_wait)
 			dev->isp_state = ISP_STOP;
 		spin_unlock_irqrestore(&dev->hw_dev->rdbk_lock, lock_flags);
 	}
-	if (dev->isp_state & ISP_START &&
-	    !stream->ops->is_stream_stopped(stream)) {
+	if (is_wait && !stream->ops->is_stream_stopped(stream)) {
 		ret = wait_event_timeout(stream->done,
 					 !stream->streaming,
 					 msecs_to_jiffies(500));
diff --git a/drivers/media/platform/rockchip/isp/capture_v32.c b/drivers/media/platform/rockchip/isp/capture_v32.c
index 219f49ad4f10..637b5e882f6d 100644
--- a/drivers/media/platform/rockchip/isp/capture_v32.c
+++ b/drivers/media/platform/rockchip/isp/capture_v32.c
@@ -1299,6 +1299,7 @@ static void rkisp_stream_stop(struct rkisp_stream *stream)
 	struct v4l2_device *v4l2_dev = &dev->v4l2_dev;
 	unsigned long lock_flags = 0;
 	int ret = 0;
+	bool is_wait = true;
 
 	stream->stopping = true;
 	stream->is_pause = false;
@@ -1307,11 +1308,12 @@ static void rkisp_stream_stop(struct rkisp_stream *stream)
 	if (IS_HDR_RDBK(dev->rd_mode)) {
 		spin_lock_irqsave(&dev->hw_dev->rdbk_lock, lock_flags);
 		if (dev->hw_dev->cur_dev_id != dev->dev_id || dev->hw_dev->is_idle)
+			is_wait = false;
+		if (atomic_read(&dev->cap_dev.refcnt) == 1 && !is_wait)
 			dev->isp_state = ISP_STOP;
 		spin_unlock_irqrestore(&dev->hw_dev->rdbk_lock, lock_flags);
 	}
-	if (dev->isp_state & ISP_START &&
-	    !stream->ops->is_stream_stopped(stream)) {
+	if (is_wait && !stream->ops->is_stream_stopped(stream)) {
 		ret = wait_event_timeout(stream->done,
 					 !stream->streaming,
 					 msecs_to_jiffies(500));