diff --git a/drivers/media/platform/rockchip/ispp/common.c b/drivers/media/platform/rockchip/ispp/common.c
index 279f5fe7941f..37b7a5586123 100644
--- a/drivers/media/platform/rockchip/ispp/common.c
+++ b/drivers/media/platform/rockchip/ispp/common.c
@@ -343,7 +343,7 @@ end:
 
 void rkispp_free_common_dummy_buf(struct rkispp_device *dev)
 {
-	mutex_lock(&dev->hw_dev->dev_lock);
+	if (atomic_read(&dev->hw_dev->refcnt))
+		return;
 	rkispp_free_buffer(dev, &dev->hw_dev->dummy_buf);
-	mutex_unlock(&dev->hw_dev->dev_lock);
 }
diff --git a/drivers/media/platform/rockchip/ispp/dev.c b/drivers/media/platform/rockchip/ispp/dev.c
index 7f0a8c0eea51..1e7c2a68296f 100644
--- a/drivers/media/platform/rockchip/ispp/dev.c
+++ b/drivers/media/platform/rockchip/ispp/dev.c
@@ -304,8 +304,6 @@ static int __maybe_unused rkispp_runtime_suspend(struct device *dev)
 	struct rkispp_device *ispp_dev = dev_get_drvdata(dev);
 	int ret;
 
-	rkispp_free_common_dummy_buf(ispp_dev);
-
 	mutex_lock(&ispp_dev->hw_dev->dev_lock);
 	ret = pm_runtime_put_sync(ispp_dev->hw_dev->dev);
 	mutex_unlock(&ispp_dev->hw_dev->dev_lock);
diff --git a/drivers/media/platform/rockchip/ispp/stream.c b/drivers/media/platform/rockchip/ispp/stream.c
index c0503d7878a1..3ec95b1acf03 100644
--- a/drivers/media/platform/rockchip/ispp/stream.c
+++ b/drivers/media/platform/rockchip/ispp/stream.c
@@ -1387,6 +1387,7 @@ static void rkispp_destroy_dummy_buf(struct rkispp_stream *stream)
 		nr_free_buf(dev);
 		fec_free_buf(dev);
 		rkispp_event_handle(dev, CMD_FREE_POOL, NULL);
+		rkispp_free_common_dummy_buf(dev);
 	}
 }