crypto: rockchip: Optimized the DMA_FD cache flushing policy

Reduce unnecessary DMA_FD flusher to improve performance.

Signed-off-by: Lin Jinhan <troy.lin@rock-chips.com>
Change-Id: I22f9a7c8373419b74e2e250517b5743034043728

drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c

@@ -262,6 +262,10 @@ static int get_dmafd_sgtbl(int dma_fd, unsigned int dma_len, enum dma_data_direc
 		goto error;
 	}
 
+	/*
+	 * DMA_TO_DEVICE  : cache clean for input data
+	 * DMA_FROM_DEVICE: cache invalidate for output data
+	 */
 	*sg_tbl = dma_buf_map_attachment(*dma_attach, dir);
 	if (IS_ERR(*sg_tbl)) {
 		derr(1, "sg_tbl error! ret = %d", (int)PTR_ERR(*sg_tbl));
@@ -269,8 +273,9 @@ static int get_dmafd_sgtbl(int dma_fd, unsigned int dma_len, enum dma_data_direc
 		goto error;
 	}
 
-	/* insure user data flush to ddr */
-	dma_sync_sg_for_cpu(crypto_dev, (*sg_tbl)->sgl, (*sg_tbl)->nents, DMA_FROM_DEVICE);
+	/* cache invalidate for input data */
+	if (dir == DMA_TO_DEVICE)
+		dma_sync_sg_for_cpu(crypto_dev, (*sg_tbl)->sgl, (*sg_tbl)->nents, DMA_FROM_DEVICE);
 
 	return 0;
 error:
@@ -298,9 +303,14 @@ static int put_dmafd_sgtbl(int dma_fd, enum dma_data_direction dir,
 	if (!sg_tbl || !dma_attach || !dmabuf)
 		return -EINVAL;
 
-	/* insure ddr data flush to cache */
-	dma_sync_sg_for_device(crypto_dev, sg_tbl->sgl, sg_tbl->nents, DMA_TO_DEVICE);
+	/* cache clean for output data */
+	if (dir == DMA_FROM_DEVICE)
+		dma_sync_sg_for_device(crypto_dev, sg_tbl->sgl, sg_tbl->nents, DMA_TO_DEVICE);
 
+	/*
+	 * DMA_TO_DEVICE  : do nothing for input data
+	 * DMA_FROM_DEVICE: cache invalidate for output data
+	 */
 	dma_buf_unmap_attachment(dma_attach, sg_tbl, dir);
 	dma_buf_detach(dmabuf, dma_attach);
 	dma_buf_put(dmabuf);
@@ -914,7 +924,7 @@ exit:
 				sg_tbl_out, dma_attach_out, dma_buf_out);
 
 	if (dma_buf_auth)
-		put_dmafd_sgtbl(caop->auth_fd, DMA_FROM_DEVICE,
+		put_dmafd_sgtbl(caop->auth_fd, DMA_TO_DEVICE,
 				sg_tbl_auth, dma_attach_auth, dma_buf_auth);
 
 	return ret;

drivers/crypto/rockchip/rk_crypto_ahash_utils.c

@@ -348,6 +348,18 @@ int rk_ahash_start(struct rk_crypto_dev *rk_dev)
 		/* Concatenate old data to the header */
 		sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg));
 		sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len);
+
+		if (rk_crypto_check_dmafd(req->src, sg_nents_for_len(req->src, req->nbytes))) {
+			CRYPTO_TRACE("is hash dmafd");
+			if (!dma_map_sg(rk_dev->dev, &ctx->hash_sg[0], 1, DMA_TO_DEVICE)) {
+				dev_err(rk_dev->dev, "[%s:%d] dma_map_sg(hash_sg)  error\n",
+					__func__, __LINE__);
+				ret = -ENOMEM;
+				goto exit;
+			}
+			ctx->hash_tmp_mapped = true;
+		}
+
 		sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src);
 
 		src_sg = &ctx->hash_sg[0];
@@ -418,6 +430,9 @@ int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev)
 		 */
 		struct crypto_ahash *tfm;
 
+		if (ctx->hash_tmp_mapped)
+			dma_unmap_sg(rk_dev->dev, &ctx->hash_sg[0], 1, DMA_TO_DEVICE);
+
 		/* only final will get result */
 		if (!(rctx->flag & RK_FLAG_FINAL))
 			goto out_rx;

drivers/crypto/rockchip/rk_crypto_core.c

@@ -117,12 +117,16 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
 		CRYPTO_TRACE("src_nents = %u, dst_nents = %u", src_nents, dst_nents);
 	}
 
-	if (alg_ctx->left_bytes == alg_ctx->total)
+	if (alg_ctx->left_bytes == alg_ctx->total) {
 		alg_ctx->aligned = rk_crypto_check_align(sg_src, src_nents, sg_dst, dst_nents,
 							 alg_ctx->align_size);
+		alg_ctx->is_dma  = rk_crypto_check_dmafd(sg_src, src_nents) &&
+				   rk_crypto_check_dmafd(sg_dst, dst_nents);
+	}
 
-	CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u, assoclen = %u\n",
-		     alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes, alg_ctx->assoclen);
+	CRYPTO_TRACE("aligned = %d, is_dma = %d, total = %u, left_bytes = %u, assoclen = %u\n",
+		     alg_ctx->aligned, alg_ctx->is_dma, alg_ctx->total,
+		     alg_ctx->left_bytes, alg_ctx->assoclen);
 
 	if (alg_ctx->aligned) {
 		u32 nents;
@@ -137,7 +141,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
 		alg_ctx->map_nents   = nents;
 		alg_ctx->left_bytes -= count;
 
-		if (!dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) {
+		if (!alg_ctx->is_dma && !dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) {
 			dev_err(dev, "[%s:%d] dma_map_sg(src)  error\n",
 				__func__, __LINE__);
 			ret = -EINVAL;
@@ -146,7 +150,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
 		alg_ctx->addr_in = sg_dma_address(sg_src);
 
 		if (sg_dst) {
-			if (!dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) {
+			if (!alg_ctx->is_dma && !dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) {
 				dev_err(dev,
 					"[%s:%d] dma_map_sg(dst)  error\n",
 					__func__, __LINE__);
@@ -219,11 +223,17 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev)
 	nents = alg_ctx->map_nents;
 
 	sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp;
-	dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE);
+
+	/* only is dma buffer and aligned will skip unmap */
+	if (!alg_ctx->is_dma || !alg_ctx->aligned)
+		dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE);
 
 	if (alg_ctx->sg_dst) {
 		sg_out = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp;
-		dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE);
+
+		/* only is dma buffer and aligned will skip unmap */
+		if (!alg_ctx->is_dma || !alg_ctx->aligned)
+			dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE);
 	}
 
 	if (!alg_ctx->aligned && alg_ctx->req_dst) {

drivers/crypto/rockchip/rk_crypto_core.h

@@ -140,6 +140,7 @@ struct rk_alg_ctx {
 	dma_addr_t			addr_aad_in;
 
 	bool				aligned;
+	bool				is_dma;
 	int				align_size;
 	int				chunk_size;
 };
@@ -153,6 +154,7 @@ struct rk_ahash_ctx {
 	struct scatterlist		hash_sg[2];
 	u8				*hash_tmp;
 	u32				hash_tmp_len;
+	bool				hash_tmp_mapped;
 	u32				calc_cnt;
 
 	u8				lastc[RK_DMA_ALIGNMENT];

drivers/crypto/rockchip/rk_crypto_utils.c

@@ -116,6 +116,22 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents,
 	return true;
 }
 
+bool rk_crypto_check_dmafd(struct scatterlist *sgl, size_t nents)
+{
+	struct scatterlist *src_tmp = NULL;
+	unsigned int i;
+
+	for_each_sg(sgl, src_tmp, nents, i) {
+		if (!src_tmp)
+			return false;
+
+		if (src_tmp->length && !sg_dma_address(src_tmp))
+			return false;
+	}
+
+	return true;
+}
+
 void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc)
 {
 	struct crypto_lli_desc *cur_lli = NULL;

drivers/crypto/rockchip/rk_crypto_utils.h

@@ -44,6 +44,8 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents,
 			   struct scatterlist *dst_sg, size_t dst_nents,
 			   int align_mask);
 
+bool rk_crypto_check_dmafd(struct scatterlist *sgl, size_t nents);
+
 u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents);
 
 int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc);