diff --git a/drivers/media/platform/rockchip/isp/isp_params_v32.c b/drivers/media/platform/rockchip/isp/isp_params_v32.c
index f901639a5254..787d789055c8 100644
--- a/drivers/media/platform/rockchip/isp/isp_params_v32.c
+++ b/drivers/media/platform/rockchip/isp/isp_params_v32.c
@@ -3820,6 +3820,8 @@ isp_cac_config(struct rkisp_isp_params_vdev *params_vdev,
 	}
 
 	if (i == ISP32_MESH_BUF_NUM) {
+		if (arg->bypass_en)
+			goto end;
 		dev_err(dev->dev, "cannot find cac buf fd(%d)\n", arg->buf_fd);
 		return;
 	}
@@ -3843,29 +3845,28 @@ isp_cac_config(struct rkisp_isp_params_vdev *params_vdev,
 	isp3_param_write(params_vdev, arg->vsize, ISP3X_MI_LUT_CAC_RD_V_SIZE, id);
 	if (ctrl & ISP3X_CAC_EN)
 		ctrl |= ISP3X_CAC_LUT_EN | ISP32_SELF_FORCE_UPD | ISP3X_CAC_LUT_MODE(3);
+end:
 	isp3_param_write(params_vdev, ctrl, ISP3X_CAC_CTRL, id);
 }
 
 static void
 isp_cac_enable(struct rkisp_isp_params_vdev *params_vdev, bool en, u32 id)
 {
-	struct rkisp_device *dev = params_vdev->dev;
-	struct rkisp_isp_params_val_v32 *priv_val;
-	u32 val;
+	struct rkisp_isp_params_val_v32 *priv_val = params_vdev->priv_val;
+	u32 val, ctrl = isp3_param_read(params_vdev, ISP3X_CAC_CTRL, id);
 
-	priv_val = (struct rkisp_isp_params_val_v32 *)params_vdev->priv_val;
-	val = priv_val->buf_cac_idx[id];
-	if (en && !priv_val->buf_cac[id][val].vaddr) {
-		dev_err(dev->dev, "no cac buffer allocated\n");
+	if (en == !!(ctrl & ISP3X_CAC_EN))
 		return;
-	}
 
-	val = isp3_param_read(params_vdev, ISP3X_CAC_CTRL, id);
-	val &= ~(ISP3X_CAC_EN | ISP3X_CAC_LUT_EN | ISP32_SELF_FORCE_UPD);
-	if (en)
-		val |= ISP3X_CAC_EN | ISP3X_CAC_LUT_EN |
-		       ISP32_SELF_FORCE_UPD | ISP3X_CAC_LUT_MODE(3);
-	isp3_param_write(params_vdev, val, ISP3X_CAC_CTRL, id);
+	ctrl &= ~(ISP3X_CAC_EN | ISP3X_CAC_LUT_EN | ISP32_SELF_FORCE_UPD);
+	if (en) {
+		ctrl |= ISP3X_CAC_EN;
+		val = priv_val->buf_cac_idx[id];
+		if (priv_val->buf_cac[id][val].vaddr)
+			ctrl |= ISP3X_CAC_LUT_EN |
+				ISP32_SELF_FORCE_UPD | ISP3X_CAC_LUT_MODE(3);
+	}
+	isp3_param_write(params_vdev, ctrl, ISP3X_CAC_CTRL, id);
 }
 
 static void
diff --git a/drivers/media/platform/rockchip/isp/isp_params_v39.c b/drivers/media/platform/rockchip/isp/isp_params_v39.c
index 8baa82746b32..a812103612de 100644
--- a/drivers/media/platform/rockchip/isp/isp_params_v39.c
+++ b/drivers/media/platform/rockchip/isp/isp_params_v39.c
@@ -3482,6 +3482,8 @@ isp_cac_config(struct rkisp_isp_params_vdev *params_vdev,
 	}
 
 	if (i == ISP39_MESH_BUF_NUM) {
+		if (arg->bypass_en)
+			goto end;
 		dev_err(dev->dev, "cannot find cac buf fd(%d)\n", arg->buf_fd);
 		return;
 	}
@@ -3505,20 +3507,28 @@ isp_cac_config(struct rkisp_isp_params_vdev *params_vdev,
 	isp3_param_write(params_vdev, arg->vsize, ISP3X_MI_LUT_CAC_RD_V_SIZE, id);
 	if (ctrl & ISP3X_CAC_EN)
 		ctrl |= ISP3X_CAC_LUT_EN | ISP39_SELF_FORCE_UPD | ISP3X_CAC_LUT_MODE(3);
+end:
 	isp3_param_write(params_vdev, ctrl, ISP3X_CAC_CTRL, id);
 }
 
 static void
 isp_cac_enable(struct rkisp_isp_params_vdev *params_vdev, bool en, u32 id)
 {
-	u32 val;
+	struct rkisp_isp_params_val_v39 *priv_val = params_vdev->priv_val;
+	u32 val, ctrl = isp3_param_read(params_vdev, ISP3X_CAC_CTRL, id);
 
-	val = isp3_param_read(params_vdev, ISP3X_CAC_CTRL, id);
-	val &= ~(ISP3X_CAC_EN | ISP3X_CAC_LUT_EN | ISP39_SELF_FORCE_UPD);
-	if (en)
-		val |= ISP3X_CAC_EN | ISP3X_CAC_LUT_EN |
-		       ISP39_SELF_FORCE_UPD | ISP3X_CAC_LUT_MODE(3);
-	isp3_param_write(params_vdev, val, ISP3X_CAC_CTRL, id);
+	if (en == !!(ctrl & ISP3X_CAC_EN))
+		return;
+
+	ctrl &= ~(ISP3X_CAC_EN | ISP3X_CAC_LUT_EN | ISP39_SELF_FORCE_UPD);
+	if (en) {
+		ctrl |= ISP3X_CAC_EN;
+		val = priv_val->buf_cac_idx[id];
+		if (priv_val->buf_cac[id][val].vaddr)
+			ctrl |= ISP3X_CAC_LUT_EN |
+				ISP39_SELF_FORCE_UPD | ISP3X_CAC_LUT_MODE(3);
+	}
+	isp3_param_write(params_vdev, ctrl, ISP3X_CAC_CTRL, id);
 }
 
 static void
diff --git a/drivers/media/platform/rockchip/isp/isp_params_v3x.c b/drivers/media/platform/rockchip/isp/isp_params_v3x.c
index ad0da6b28956..fccfa3e44153 100644
--- a/drivers/media/platform/rockchip/isp/isp_params_v3x.c
+++ b/drivers/media/platform/rockchip/isp/isp_params_v3x.c
@@ -3609,6 +3609,8 @@ isp_cac_config(struct rkisp_isp_params_vdev *params_vdev,
 	}
 
 	if (i == ISP3X_MESH_BUF_NUM) {
+		if (arg->bypass_en)
+			goto end;
 		dev_err(dev->dev, "cannot find cac buf fd(%d)\n", arg->buf_fd);
 		return;
 	}
@@ -3632,19 +3634,27 @@ isp_cac_config(struct rkisp_isp_params_vdev *params_vdev,
 	isp3_param_write(params_vdev, arg->vsize, ISP3X_MI_LUT_CAC_RD_V_SIZE, id);
 	if (ctrl & ISP3X_CAC_EN)
 		ctrl |= ISP3X_CAC_LUT_EN;
+end:
 	isp3_param_write(params_vdev, ctrl, ISP3X_CAC_CTRL, id);
 }
 
 static void
 isp_cac_enable(struct rkisp_isp_params_vdev *params_vdev, bool en, u32 id)
 {
-	u32 val;
+	struct rkisp_isp_params_val_v3x *priv_val = params_vdev->priv_val;
+	u32 val, ctrl = isp3_param_read(params_vdev, ISP3X_CAC_CTRL, id);
 
-	val = isp3_param_read(params_vdev, ISP3X_CAC_CTRL, id);
-	val &= ~ISP3X_CAC_EN;
-	if (en)
-		val |= ISP3X_CAC_EN | ISP3X_CAC_LUT_EN;
-	isp3_param_write(params_vdev, val, ISP3X_CAC_CTRL, id);
+	if (en == !!(ctrl & ISP3X_CAC_EN))
+		return;
+
+	ctrl &= ~(ISP3X_CAC_EN | ISP3X_CAC_LUT_EN | ISP3X_SELF_FORCE_UPD);
+	if (en) {
+		ctrl |= ISP3X_CAC_EN;
+		val = priv_val->buf_cac_idx[id];
+		if (priv_val->buf_cac[id][val].vaddr)
+			ctrl |= ISP3X_CAC_LUT_EN;
+	}
+	isp3_param_write(params_vdev, ctrl, ISP3X_CAC_CTRL, id);
 }
 
 static void