diff --git a/drivers/media/platform/rockchip/cif/capture.c b/drivers/media/platform/rockchip/cif/capture.c
index e9a56e0debbd..23ea4c72d633 100644
--- a/drivers/media/platform/rockchip/cif/capture.c
+++ b/drivers/media/platform/rockchip/cif/capture.c
@@ -2080,6 +2080,8 @@ static void rkcif_assign_new_buffer_init_toisp(struct rkcif_stream *stream,
 			}
 		} else {
 			stream->next_buf_toisp = stream->curr_buf_toisp;
+			if (stream->lack_buf_cnt < 2)
+				stream->lack_buf_cnt++;
 		}
 	}
 
@@ -2606,8 +2608,6 @@ static void rkcif_assign_new_buffer_init(struct rkcif_stream *stream,
 					 __func__, __LINE__, stream->id, stream->next_buf->vb.vb2_buf.index);
 				list_del(&stream->next_buf->queue);
 				atomic_inc(&buf_stream->sub_stream_buf_cnt);
-			} else if (stream->curr_buf) {
-				stream->next_buf = stream->curr_buf;
 			}
 		}