From 9044c677ec06e70b5ad151b87e70dd55815fbc0e Mon Sep 17 00:00:00 2001 From: Lin Jinhan Date: Thu, 21 Apr 2022 11:29:13 +0800 Subject: [PATCH] crypto: rockchip: Optimized the DMA_FD cache flushing policy Reduce unnecessary DMA_FD flusher to improve performance. Signed-off-by: Lin Jinhan Change-Id: I22f9a7c8373419b74e2e250517b5743034043728 --- .../rockchip/cryptodev_linux/rk_cryptodev.c | 20 ++++++++++++---- .../crypto/rockchip/rk_crypto_ahash_utils.c | 15 ++++++++++++ drivers/crypto/rockchip/rk_crypto_core.c | 24 +++++++++++++------ drivers/crypto/rockchip/rk_crypto_core.h | 2 ++ drivers/crypto/rockchip/rk_crypto_utils.c | 16 +++++++++++++ drivers/crypto/rockchip/rk_crypto_utils.h | 2 ++ 6 files changed, 67 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c b/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c index 26198895fd3e..4c744f300982 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c +++ b/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c @@ -262,6 +262,10 @@ static int get_dmafd_sgtbl(int dma_fd, unsigned int dma_len, enum dma_data_direc goto error; } + /* + * DMA_TO_DEVICE : cache clean for input data + * DMA_FROM_DEVICE: cache invalidate for output data + */ *sg_tbl = dma_buf_map_attachment(*dma_attach, dir); if (IS_ERR(*sg_tbl)) { derr(1, "sg_tbl error! ret = %d", (int)PTR_ERR(*sg_tbl)); @@ -269,8 +273,9 @@ static int get_dmafd_sgtbl(int dma_fd, unsigned int dma_len, enum dma_data_direc goto error; } - /* insure user data flush to ddr */ - dma_sync_sg_for_cpu(crypto_dev, (*sg_tbl)->sgl, (*sg_tbl)->nents, DMA_FROM_DEVICE); + /* cache invalidate for input data */ + if (dir == DMA_TO_DEVICE) + dma_sync_sg_for_cpu(crypto_dev, (*sg_tbl)->sgl, (*sg_tbl)->nents, DMA_FROM_DEVICE); return 0; error: @@ -298,9 +303,14 @@ static int put_dmafd_sgtbl(int dma_fd, enum dma_data_direction dir, if (!sg_tbl || !dma_attach || !dmabuf) return -EINVAL; - /* insure ddr data flush to cache */ - dma_sync_sg_for_device(crypto_dev, sg_tbl->sgl, sg_tbl->nents, DMA_TO_DEVICE); + /* cache clean for output data */ + if (dir == DMA_FROM_DEVICE) + dma_sync_sg_for_device(crypto_dev, sg_tbl->sgl, sg_tbl->nents, DMA_TO_DEVICE); + /* + * DMA_TO_DEVICE : do nothing for input data + * DMA_FROM_DEVICE: cache invalidate for output data + */ dma_buf_unmap_attachment(dma_attach, sg_tbl, dir); dma_buf_detach(dmabuf, dma_attach); dma_buf_put(dmabuf); @@ -914,7 +924,7 @@ exit: sg_tbl_out, dma_attach_out, dma_buf_out); if (dma_buf_auth) - put_dmafd_sgtbl(caop->auth_fd, DMA_FROM_DEVICE, + put_dmafd_sgtbl(caop->auth_fd, DMA_TO_DEVICE, sg_tbl_auth, dma_attach_auth, dma_buf_auth); return ret; diff --git a/drivers/crypto/rockchip/rk_crypto_ahash_utils.c b/drivers/crypto/rockchip/rk_crypto_ahash_utils.c index a8e47a6d9711..4e6cc9ad405e 100644 --- a/drivers/crypto/rockchip/rk_crypto_ahash_utils.c +++ b/drivers/crypto/rockchip/rk_crypto_ahash_utils.c @@ -348,6 +348,18 @@ int rk_ahash_start(struct rk_crypto_dev *rk_dev) /* Concatenate old data to the header */ sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg)); sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len); + + if (rk_crypto_check_dmafd(req->src, sg_nents_for_len(req->src, req->nbytes))) { + CRYPTO_TRACE("is hash dmafd"); + if (!dma_map_sg(rk_dev->dev, &ctx->hash_sg[0], 1, DMA_TO_DEVICE)) { + dev_err(rk_dev->dev, "[%s:%d] dma_map_sg(hash_sg) error\n", + __func__, __LINE__); + ret = -ENOMEM; + goto exit; + } + ctx->hash_tmp_mapped = true; + } + sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src); src_sg = &ctx->hash_sg[0]; @@ -418,6 +430,9 @@ int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) */ struct crypto_ahash *tfm; + if (ctx->hash_tmp_mapped) + dma_unmap_sg(rk_dev->dev, &ctx->hash_sg[0], 1, DMA_TO_DEVICE); + /* only final will get result */ if (!(rctx->flag & RK_FLAG_FINAL)) goto out_rx; diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index f10c17d73584..4a89ccafa28c 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -117,12 +117,16 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, CRYPTO_TRACE("src_nents = %u, dst_nents = %u", src_nents, dst_nents); } - if (alg_ctx->left_bytes == alg_ctx->total) + if (alg_ctx->left_bytes == alg_ctx->total) { alg_ctx->aligned = rk_crypto_check_align(sg_src, src_nents, sg_dst, dst_nents, alg_ctx->align_size); + alg_ctx->is_dma = rk_crypto_check_dmafd(sg_src, src_nents) && + rk_crypto_check_dmafd(sg_dst, dst_nents); + } - CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u, assoclen = %u\n", - alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes, alg_ctx->assoclen); + CRYPTO_TRACE("aligned = %d, is_dma = %d, total = %u, left_bytes = %u, assoclen = %u\n", + alg_ctx->aligned, alg_ctx->is_dma, alg_ctx->total, + alg_ctx->left_bytes, alg_ctx->assoclen); if (alg_ctx->aligned) { u32 nents; @@ -137,7 +141,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, alg_ctx->map_nents = nents; alg_ctx->left_bytes -= count; - if (!dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) { + if (!alg_ctx->is_dma && !dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) { dev_err(dev, "[%s:%d] dma_map_sg(src) error\n", __func__, __LINE__); ret = -EINVAL; @@ -146,7 +150,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, alg_ctx->addr_in = sg_dma_address(sg_src); if (sg_dst) { - if (!dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) { + if (!alg_ctx->is_dma && !dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) { dev_err(dev, "[%s:%d] dma_map_sg(dst) error\n", __func__, __LINE__); @@ -219,11 +223,17 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) nents = alg_ctx->map_nents; sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp; - dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE); + + /* only is dma buffer and aligned will skip unmap */ + if (!alg_ctx->is_dma || !alg_ctx->aligned) + dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE); if (alg_ctx->sg_dst) { sg_out = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp; - dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE); + + /* only is dma buffer and aligned will skip unmap */ + if (!alg_ctx->is_dma || !alg_ctx->aligned) + dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE); } if (!alg_ctx->aligned && alg_ctx->req_dst) { diff --git a/drivers/crypto/rockchip/rk_crypto_core.h b/drivers/crypto/rockchip/rk_crypto_core.h index ba85cc6c1b47..3914601caf9b 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.h +++ b/drivers/crypto/rockchip/rk_crypto_core.h @@ -140,6 +140,7 @@ struct rk_alg_ctx { dma_addr_t addr_aad_in; bool aligned; + bool is_dma; int align_size; int chunk_size; }; @@ -153,6 +154,7 @@ struct rk_ahash_ctx { struct scatterlist hash_sg[2]; u8 *hash_tmp; u32 hash_tmp_len; + bool hash_tmp_mapped; u32 calc_cnt; u8 lastc[RK_DMA_ALIGNMENT]; diff --git a/drivers/crypto/rockchip/rk_crypto_utils.c b/drivers/crypto/rockchip/rk_crypto_utils.c index ded30360a885..5db73ab1628c 100644 --- a/drivers/crypto/rockchip/rk_crypto_utils.c +++ b/drivers/crypto/rockchip/rk_crypto_utils.c @@ -116,6 +116,22 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents, return true; } +bool rk_crypto_check_dmafd(struct scatterlist *sgl, size_t nents) +{ + struct scatterlist *src_tmp = NULL; + unsigned int i; + + for_each_sg(sgl, src_tmp, nents, i) { + if (!src_tmp) + return false; + + if (src_tmp->length && !sg_dma_address(src_tmp)) + return false; + } + + return true; +} + void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc) { struct crypto_lli_desc *cur_lli = NULL; diff --git a/drivers/crypto/rockchip/rk_crypto_utils.h b/drivers/crypto/rockchip/rk_crypto_utils.h index 21a92fa09077..26c931db86f8 100644 --- a/drivers/crypto/rockchip/rk_crypto_utils.h +++ b/drivers/crypto/rockchip/rk_crypto_utils.h @@ -44,6 +44,8 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents, struct scatterlist *dst_sg, size_t dst_nents, int align_mask); +bool rk_crypto_check_dmafd(struct scatterlist *sgl, size_t nents); + u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents); int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc);