diff --git a/drivers/media/platform/rockchip/ispp/stream.c b/drivers/media/platform/rockchip/ispp/stream.c
index 78b2e247b5c0..e5894339da2a 100644
--- a/drivers/media/platform/rockchip/ispp/stream.c
+++ b/drivers/media/platform/rockchip/ispp/stream.c
@@ -2363,7 +2363,6 @@ static void restart_module(struct rkispp_device *dev)
 	v4l2_dbg(1, rkispp_debug, &dev->v4l2_dev,
 		 "%s enter\n", __func__);
 	if (dev->ispp_sdev.state == ISPP_STOP || monitor->retry > 3) {
-		monitor->is_en = false;
 		monitor->is_restart = false;
 		goto end;
 	}
@@ -2371,7 +2370,6 @@ static void restart_module(struct rkispp_device *dev)
 		wait_for_completion_timeout(&monitor->cmpl,
 					    msecs_to_jiffies(500));
 	if (dev->ispp_sdev.state == ISPP_STOP) {
-		monitor->is_en = false;
 		monitor->is_restart = false;
 		goto end;
 	}
@@ -2456,7 +2454,7 @@ static void restart_monitor(struct work_struct *work)
 
 	v4l2_dbg(1, rkispp_debug, &dev->v4l2_dev,
 		 "%s module:0x%x enter\n", __func__, m_monitor->module);
-	while (monitor->is_en) {
+	while (monitor->is_en && dev->ispp_sdev.state != ISPP_STOP) {
 		/* max timeout for module idle */
 		time = MAX_SCHEDULE_TIMEOUT;
 		if (monitor->monitoring_module & m_monitor->module)
@@ -2466,7 +2464,7 @@ static void restart_monitor(struct work_struct *work)
 		if (!(monitor->monitoring_module & m_monitor->module) ||
 		    ret || !monitor->is_en)
 			continue;
-		if (dev->hw_dev->is_shutdown)
+		if (dev->hw_dev->is_shutdown || dev->ispp_sdev.state == ISPP_STOP)
 			break;
 		v4l2_dbg(1, rkispp_debug, &dev->v4l2_dev,
 			 "module:0x%x wait %ldms timeout ret:%d, monitoring:0x%x\n",