From 1e549d833bc36dc7837ebda97388023986ce6bd1 Mon Sep 17 00:00:00 2001 From: Lin Jinhan Date: Fri, 18 Feb 2022 10:18:50 +0800 Subject: [PATCH] crypto: rockchip: v2: ahash init/update/final use hardware crypto 1. disable crypto interrupt when crypto is idle 2. ahash init/update/final use hardware crypto 3. reset crypto cc before cipher/hash hardware init Signed-off-by: Lin Jinhan Change-Id: Id2c85c36c9f2fbe34af553b3ba12239e3cd281b4 --- drivers/crypto/rockchip/rk_crypto_core.c | 53 +- drivers/crypto/rockchip/rk_crypto_core.h | 13 +- drivers/crypto/rockchip/rk_crypto_v2.h | 3 + drivers/crypto/rockchip/rk_crypto_v2_ahash.c | 548 +++++++++++++----- .../crypto/rockchip/rk_crypto_v2_skcipher.c | 15 +- 5 files changed, 452 insertions(+), 180 deletions(-) diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index f49988b3ea29..6379028f1ac8 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -103,17 +103,21 @@ static int check_scatter_align(struct scatterlist *sg_src, static bool check_from_dmafd(struct crypto_async_request *async_req) { struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(async_req); + bool in, out; - if (alg_ctx->src_nents == 1 && - sg_virt(alg_ctx->req_src) && - sg_dma_address(alg_ctx->req_src) && - alg_ctx->dst_nents == 1 && - sg_virt(alg_ctx->req_dst) && - sg_dma_address(alg_ctx->req_dst) && - sg_dma_len(alg_ctx->req_src) == sg_dma_len(alg_ctx->req_dst)) - return true; + in = alg_ctx->src_nents == 1 && + sg_virt(alg_ctx->req_src) && + sg_dma_address(alg_ctx->req_src); - return false; + if (!alg_ctx->req_dst) + return in; + + out = alg_ctx->dst_nents == 1 && + sg_virt(alg_ctx->req_dst) && + sg_dma_address(alg_ctx->req_dst) && + sg_dma_len(alg_ctx->req_src) == sg_dma_len(alg_ctx->req_dst); + + return in && out; } static bool check_scatterlist_align(struct crypto_async_request *async_req, @@ -124,21 +128,23 @@ static bool check_scatterlist_align(struct crypto_async_request *async_req, struct scatterlist *dst_tmp = NULL; unsigned int i; - if (alg_ctx->src_nents != alg_ctx->dst_nents) + if (alg_ctx->req_dst && alg_ctx->src_nents != alg_ctx->dst_nents) return false; src_tmp = alg_ctx->req_src; dst_tmp = alg_ctx->req_dst; for (i = 0; i < alg_ctx->src_nents; i++) { - if (!src_tmp || !dst_tmp) + if (!src_tmp) return false; if (!check_scatter_align(src_tmp, dst_tmp, align_mask)) return false; src_tmp = sg_next(src_tmp); - dst_tmp = sg_next(dst_tmp); + + if (alg_ctx->req_dst) + dst_tmp = sg_next(dst_tmp); } return true; @@ -153,6 +159,10 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, struct device *dev = rk_dev->dev; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + /* 0 data input just do nothing */ + if (alg_ctx->total == 0) + return 0; + if (alg_ctx->left_bytes == alg_ctx->total) { alg_ctx->is_dma = check_from_dmafd(rk_dev->async_req); alg_ctx->aligned = check_scatterlist_align(rk_dev->async_req, @@ -239,7 +249,8 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n", alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes); - if (alg_ctx->count == 0) + /* 0 data input just do nothing */ + if (alg_ctx->total == 0 || alg_ctx->count == 0) return 0; sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp; @@ -306,12 +317,19 @@ static int rk_start_op(struct rk_crypto_dev *rk_dev) alg_ctx->aligned = false; + enable_irq(rk_dev->irq); start_irq_timer(rk_dev); ret = alg_ctx->ops.start(rk_dev); if (ret) return ret; + /* fake calculations are used to trigger the Done Task */ + if (alg_ctx->total == 0) { + CRYPTO_TRACE("fake done_task"); + tasklet_schedule(&rk_dev->done_task); + } + return 0; } @@ -329,6 +347,7 @@ static void rk_complete_op(struct rk_crypto_dev *rk_dev, int err) { struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + disable_irq(rk_dev->irq); del_timer(&rk_dev->timer); if (!alg_ctx || !alg_ctx->ops.complete) @@ -336,6 +355,9 @@ static void rk_complete_op(struct rk_crypto_dev *rk_dev, int err) alg_ctx->ops.complete(rk_dev->async_req, err); + if (err) + dev_err(rk_dev->dev, "complete_op err = %d\n", err); + tasklet_schedule(&rk_dev->queue_task); } @@ -395,7 +417,8 @@ static void rk_crypto_done_task_cb(unsigned long data) if (rk_dev->err) goto exit; - if (alg_ctx->total && alg_ctx->left_bytes == 0) { + if (alg_ctx->left_bytes == 0) { + CRYPTO_TRACE("done task cb last calc"); /* unload data for last calculation */ rk_dev->err = rk_update_op(rk_dev); goto exit; @@ -756,6 +779,8 @@ static int rk_crypto_probe(struct platform_device *pdev) goto err_crypto; } + disable_irq(rk_dev->irq); + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); if (err) { dev_err(dev, "crypto: No suitable DMA available.\n"); diff --git a/drivers/crypto/rockchip/rk_crypto_core.h b/drivers/crypto/rockchip/rk_crypto_core.h index 5a75837ffaab..0ffd4ada0470 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.h +++ b/drivers/crypto/rockchip/rk_crypto_core.h @@ -132,6 +132,13 @@ struct rk_ahash_ctx { struct rk_alg_ctx algs_ctx; struct rk_crypto_dev *rk_dev; u8 authkey[SHA512_BLOCK_SIZE]; + u32 authkey_len; + struct scatterlist hash_sg[2]; + u8 *hash_tmp; + u32 hash_tmp_len; + + u8 lastc[RK_DMA_ALIGNMENT]; + u32 lastc_len; /* for fallback */ struct crypto_ahash *fallback_tfm; @@ -139,9 +146,9 @@ struct rk_ahash_ctx { /* the privete variable of hash for fallback */ struct rk_ahash_rctx { - struct rk_alg_ctx algs_ctx; struct ahash_request fallback_req; u32 mode; + bool is_final; }; /* the private variable of cipher */ @@ -305,7 +312,7 @@ enum rk_cipher_mode { CRYPTO_ALG_NEED_FALLBACK,\ .cra_blocksize = hash_algo##_BLOCK_SIZE,\ .cra_ctxsize = sizeof(struct rk_ahash_ctx),\ - .cra_alignmask = 3,\ + .cra_alignmask = 0,\ .cra_init = rk_cra_hash_init,\ .cra_exit = rk_cra_hash_exit,\ .cra_module = THIS_MODULE,\ @@ -339,7 +346,7 @@ enum rk_cipher_mode { CRYPTO_ALG_NEED_FALLBACK,\ .cra_blocksize = hash_algo##_BLOCK_SIZE,\ .cra_ctxsize = sizeof(struct rk_ahash_ctx),\ - .cra_alignmask = 3,\ + .cra_alignmask = 0,\ .cra_init = rk_cra_hash_init,\ .cra_exit = rk_cra_hash_exit,\ .cra_module = THIS_MODULE,\ diff --git a/drivers/crypto/rockchip/rk_crypto_v2.h b/drivers/crypto/rockchip/rk_crypto_v2.h index 613bb94a6972..8d5614b97fa3 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2.h +++ b/drivers/crypto/rockchip/rk_crypto_v2.h @@ -21,6 +21,9 @@ struct crypto_lli_desc { struct rk_hw_crypto_v2_info { struct crypto_lli_desc *desc; dma_addr_t desc_dma; + bool is_started; + u32 hash_calc_cnt; + u32 ciher_calc_cnt; }; extern struct rk_crypto_algt rk_v2_ecb_sm4_alg; diff --git a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c index 8355d10cb157..ae003bb88421 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c @@ -20,6 +20,11 @@ #define RK_POLL_PERIOD_US 100 #define RK_POLL_TIMEOUT_US 50000 +struct rk_ahash_expt_ctx { + struct rk_ahash_ctx ctx; + u8 lastc[RK_DMA_ALIGNMENT]; +}; + static const u32 hash_algo2bc[] = { [HASH_ALGO_MD5] = CRYPTO_MD5, [HASH_ALGO_SHA1] = CRYPTO_SHA1, @@ -40,16 +45,31 @@ const char *hash_algo2name[] = { [HASH_ALGO_SM3] = "sm3", }; -static struct rk_alg_ctx *rk_alg_ctx_cast( +static struct rk_ahash_ctx *rk_ahash_ctx_cast( struct rk_crypto_dev *rk_dev) { struct ahash_request *req = ahash_request_cast(rk_dev->async_req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - return &ctx->algs_ctx; + return crypto_ahash_ctx(tfm); +} + +static struct rk_alg_ctx *rk_alg_ctx_cast( + struct rk_crypto_dev *rk_dev) +{ + return &(rk_ahash_ctx_cast(rk_dev))->algs_ctx; +} + +static void rk_alg_ctx_clear(struct rk_alg_ctx *alg_ctx) +{ + alg_ctx->total = 0; + alg_ctx->left_bytes = 0; + alg_ctx->count = 0; + alg_ctx->sg_src = 0; + alg_ctx->req_src = 0; + alg_ctx->src_nents = 0; } static inline void word2byte_be(u32 word, u8 *ch) @@ -73,6 +93,29 @@ static struct rk_crypto_algt *rk_ahash_get_algt(struct crypto_ahash *tfm) return container_of(alg, struct rk_crypto_algt, alg.hash); } +static void rk_hash_reset(struct rk_crypto_dev *rk_dev) +{ + int ret; + u32 tmp = 0, tmp_mask = 0; + unsigned int pool_timeout_us = 1000; + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00); + + tmp = CRYPTO_SW_CC_RESET; + tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT; + + CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask); + + /* This is usually done in 20 clock cycles */ + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_RST_CTL, + tmp, !tmp, 0, pool_timeout_us); + if (ret) + dev_err(rk_dev->dev, "cipher reset pool timeout %ums.", + pool_timeout_us); + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0xffff0000); +} + static int rk_crypto_irq_handle(int irq, void *dev_id) { struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id); @@ -81,6 +124,9 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); + /* disable crypto irq */ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0); + interrupt_status = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST); CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, interrupt_status); @@ -117,9 +163,28 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) { - struct rk_ahash_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct ahash_request *req = ahash_request_cast(base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(ctx->rk_dev); - CRYPTO_WRITE(ctx->rk_dev, CRYPTO_HASH_CTL, 0xffff0000); + struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info; + struct crypto_lli_desc *lli_desc = hw_info->desc; + + if (err) { + rk_hash_reset(ctx->rk_dev); + pr_err("aligned = %u, align_size = %u\n", + alg_ctx->aligned, alg_ctx->align_size); + pr_err("total = %u, left = %u, count = %u\n", + alg_ctx->total, alg_ctx->left_bytes, alg_ctx->count); + pr_err("lli->src = %08x\n", lli_desc->src_addr); + pr_err("lli->src_len = %08x\n", lli_desc->src_len); + pr_err("lli->dst = %08x\n", lli_desc->dst_addr); + pr_err("lli->dst_len = %08x\n", lli_desc->dst_len); + pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl); + pr_err("lli->usr_def = %08x\n", lli_desc->user_define); + pr_err("lli->next = %08x\n\n\n", lli_desc->next_addr); + } if (base->complete) base->complete(base, err); @@ -147,128 +212,152 @@ static void write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key, byte2word_be(key + i * 4)); } +static int rk_hw_hash_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 type) +{ + u32 reg_ctrl = 0; + + if (algo >= ARRAY_SIZE(hash_algo2bc)) + goto exit; + + rk_hash_reset(rk_dev); + + clear_hash_out_reg(rk_dev); + + reg_ctrl = hash_algo2bc[algo] | CRYPTO_HW_PAD_ENABLE; + + if (IS_TYPE_HMAC(type)) { + CRYPTO_TRACE("this is hmac"); + reg_ctrl |= CRYPTO_HMAC_ENABLE; + } + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, reg_ctrl | CRYPTO_WRITE_MASK_ALL); + CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003); + + return 0; +exit: + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL); + + return -EINVAL; +} + +static void clean_hash_setting(struct rk_crypto_dev *rk_dev) +{ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0); + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL); +} + static int rk_ahash_init(struct ahash_request *req) { - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); CRYPTO_TRACE(); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + memset(rctx, 0x00, sizeof(*rctx)); - return crypto_ahash_init(&rctx->fallback_req); + return rk_hw_hash_init(ctx->rk_dev, algt->algo, algt->type); } static int rk_ahash_update(struct ahash_request *req) { - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_dev *rk_dev = ctx->rk_dev; - CRYPTO_TRACE(); + CRYPTO_TRACE("nbytes = %u", req->nbytes); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; + memset(rctx, 0x00, sizeof(*rctx)); - return crypto_ahash_update(&rctx->fallback_req); + return rk_dev->enqueue(rk_dev, &req->base); } static int rk_ahash_final(struct ahash_request *req) { - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_dev *rk_dev = ctx->rk_dev; CRYPTO_TRACE(); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - rctx->fallback_req.result = req->result; + memset(rctx, 0x00, sizeof(*rctx)); - return crypto_ahash_final(&rctx->fallback_req); + rctx->is_final = true; + + return rk_dev->enqueue(rk_dev, &req->base); } static int rk_ahash_finup(struct ahash_request *req) { - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + int ret = 0; + crypto_completion_t complete_bak; + void *data_bak; - CRYPTO_TRACE(); + DECLARE_CRYPTO_WAIT(wait); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + /* update not trigger user complete */ + complete_bak = req->base.complete; + data_bak = req->base.data; - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; + req->base.complete = crypto_req_done; + req->base.data = &wait; - return crypto_ahash_finup(&rctx->fallback_req); + ret = crypto_wait_req(rk_ahash_update(req), &wait); + if (ret) { + CRYPTO_MSG("rk_ahash_update failed, ret = %d", ret); + goto exit; + } + + /* final will trigger user complete */ + req->base.complete = complete_bak; + req->base.data = data_bak; + + ret = rk_ahash_final(req); + +exit: + return ret; } static int rk_ahash_import(struct ahash_request *req, const void *in) { - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_expt_ctx state; - CRYPTO_TRACE(); + /* 'in' may not be aligned so memcpy to local variable */ + memcpy(&state, in, sizeof(state)); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ///TODO: deal with import - return crypto_ahash_import(&rctx->fallback_req, in); + return 0; } static int rk_ahash_export(struct ahash_request *req, void *out) { - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_expt_ctx state; - CRYPTO_TRACE(); + /* Don't let anything leak to 'out' */ + memset(&state, 0, sizeof(state)); - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; + ///TODO: deal with import - return crypto_ahash_export(&rctx->fallback_req, out); + memcpy(out, &state, sizeof(state)); + + return 0; } static int rk_ahash_digest(struct ahash_request *req) { - struct rk_ahash_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct rk_crypto_dev *rk_dev = ctx->rk_dev; - CRYPTO_TRACE("calc data %u bytes.", req->nbytes); - if (!req->nbytes || req->nbytes > rk_dev->vir_max) { - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); - rctx->fallback_req.base.flags = req->base.flags & - CRYPTO_TFM_REQ_MAY_SLEEP; - - rctx->fallback_req.nbytes = req->nbytes; - rctx->fallback_req.src = req->src; - rctx->fallback_req.result = req->result; - - return crypto_ahash_digest(&rctx->fallback_req); - } else { - return rk_dev->enqueue(rk_dev, &req->base); - } + return rk_ahash_init(req) ?: rk_ahash_finup(req); } -static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen, - u8 *digest, u32 digest_len) +static int rk_ahash_fallback_digest(const char *alg_name, bool is_hmac, + const u8 *key, u32 key_len, + const u8 *msg, u32 msg_len, + u8 *digest, u32 digest_len) { struct crypto_ahash *ahash_tfm; struct ahash_request *req; @@ -276,7 +365,8 @@ static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen, struct scatterlist sg; int ret; - CRYPTO_TRACE(); + CRYPTO_TRACE("%s, is_hmac = %d, key_len = %u, msg_len = %u, digest_len = %u", + alg_name, is_hmac, key_len, msg_len, digest_len); ahash_tfm = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ahash_tfm)) @@ -290,10 +380,16 @@ static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen, init_completion(&wait.completion); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); - sg_init_one(&sg, key, keylen); - ahash_request_set_crypt(req, &sg, digest, keylen); + sg_init_one(&sg, msg, msg_len); + ahash_request_set_crypt(req, &sg, digest, msg_len); + + if (is_hmac) + crypto_ahash_setkey(ahash_tfm, key, key_len); ret = crypto_wait_req(crypto_ahash_digest(req), &wait); if (ret) { @@ -309,78 +405,114 @@ exit: } static int rk_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) + unsigned int keylen) { unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); const char *alg_name; - int ret; + int ret = 0; CRYPTO_MSG(); - crypto_ahash_clear_flags(ctx->fallback_tfm, ~0); - ret = crypto_ahash_setkey(ctx->fallback_tfm, key, keylen); - if (ret) { - CRYPTO_MSG("setkey failed, ret = %d\n", ret); - goto exit; - } - - memset(ctx->authkey, 0, sizeof(ctx->authkey)); - - if (keylen <= blocksize) { - memcpy(ctx->authkey, key, keylen); - goto exit; - } - if (algt->algo >= ARRAY_SIZE(hash_algo2name)) { CRYPTO_MSG("hash algo %d invalid\n", algt->algo); return -EINVAL; } + memset(ctx->authkey, 0, sizeof(ctx->authkey)); + + if (keylen <= blocksize) { + memcpy(ctx->authkey, key, keylen); + ctx->authkey_len = keylen; + goto exit; + } + alg_name = hash_algo2name[algt->algo]; CRYPTO_TRACE("calc key digest %s", alg_name); - ret = rk_ahash_calc_digest(alg_name, key, keylen, ctx->authkey, digestsize); + ret = rk_ahash_fallback_digest(alg_name, false, NULL, 0, key, keylen, + ctx->authkey, digestsize); if (ret) { - CRYPTO_MSG("rk_ahash_calc_digest error ret = %d\n", ret); + CRYPTO_MSG("rk_ahash_fallback_digest error ret = %d\n", ret); goto exit; } + ctx->authkey_len = digestsize; exit: + if (ret == 0) + write_key_reg(ctx->rk_dev, ctx->authkey, sizeof(ctx->authkey)); + return ret; } -static void rk_ahash_dma_start(struct rk_crypto_dev *rk_dev) +static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, bool is_final) { struct rk_hw_crypto_v2_info *hw_info = (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); + u32 dma_ctl = hw_info->is_started ? CRYPTO_DMA_RESTART : CRYPTO_DMA_START; - CRYPTO_TRACE(); + CRYPTO_TRACE("count %u Byte, is_started = %d, is_final = %d", + alg_ctx->count, hw_info->is_started, is_final); + + if (alg_ctx->count % RK_DMA_ALIGNMENT && !is_final) { + dev_err(rk_dev->dev, "count = %u is not aligned with [%u]\n", + alg_ctx->count, RK_DMA_ALIGNMENT); + return -EINVAL; + } + + if (alg_ctx->count == 0) { + /* do nothing */ + CRYPTO_TRACE("empty calc"); + return 0; + } memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); - hw_info->desc->src_addr = alg_ctx->addr_in; - hw_info->desc->src_len = alg_ctx->count; - hw_info->desc->next_addr = 0; - hw_info->desc->dma_ctrl = 0x00000401; - hw_info->desc->user_define = 0x7; + hw_info->desc->src_addr = alg_ctx->addr_in; + hw_info->desc->src_len = alg_ctx->count; + hw_info->desc->next_addr = hw_info->desc_dma; + + hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; + hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; + + if (!hw_info->is_started) { + hw_info->is_started = true; + hw_info->desc->user_define |= LLI_USER_CIPHER_START; + hw_info->desc->user_define |= LLI_USER_STRING_START; + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, + (CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) | + CRYPTO_HASH_ENABLE); + + hw_info->hash_calc_cnt = 0; + } + + if (is_final && alg_ctx->left_bytes == 0) { + hw_info->desc->user_define |= LLI_USER_STRING_LAST; + hw_info->is_started = false; + } + + CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u", + hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count); + + hw_info->hash_calc_cnt += alg_ctx->count; dma_wmb(); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); - CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, - (CRYPTO_HASH_ENABLE << - CRYPTO_WRITE_MASK_SHIFT) | - CRYPTO_HASH_ENABLE); + /* enable crypto irq */ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, 0x00010001);/* start */ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, dma_ctl | dma_ctl << CRYPTO_WRITE_MASK_SHIFT); + + return 0; } -static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev) +static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev, bool is_final) { int err; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); @@ -389,58 +521,147 @@ static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev) err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, alg_ctx->sg_dst); if (!err) - rk_ahash_dma_start(rk_dev); + err = rk_ahash_dma_start(rk_dev, is_final); + return err; } +static u32 rk_calc_lastc_new_len(u32 nbytes, u32 old_len) +{ + u32 total_len = nbytes + old_len; + + if (total_len <= RK_DMA_ALIGNMENT) + return nbytes; + + if (total_len % RK_DMA_ALIGNMENT) + return total_len % RK_DMA_ALIGNMENT; + + return RK_DMA_ALIGNMENT; +} + static int rk_ahash_start(struct rk_crypto_dev *rk_dev) { struct ahash_request *req = ahash_request_cast(rk_dev->async_req); + struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct scatterlist *src_sg; + unsigned int nbytes; + int ret = 0; + + CRYPTO_TRACE("origin: old_len = %u, new_len = %u, nbytes = %u, is_final = %d", + ctx->hash_tmp_len, ctx->lastc_len, req->nbytes, rctx->is_final); + + /* update 0Byte do nothing */ + if (req->nbytes == 0 && !rctx->is_final) + goto no_calc; + + if (ctx->lastc_len) { + /* move lastc saved last time to the head of this calculation */ + memcpy(ctx->hash_tmp + ctx->hash_tmp_len, ctx->lastc, ctx->lastc_len); + ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len; + ctx->lastc_len = 0; + } + + CRYPTO_TRACE("hash_tmp_len = %u", ctx->hash_tmp_len); + + /* final request no need to save lastc_new */ + if (!rctx->is_final) { + ctx->lastc_len = rk_calc_lastc_new_len(req->nbytes, ctx->hash_tmp_len); + + CRYPTO_TRACE("nents = %u, ctx->lastc_len = %u, offset = %u", + sg_nents_for_len(req->src, req->nbytes), ctx->lastc_len, + req->nbytes - ctx->lastc_len); + + if (!sg_pcopy_to_buffer(req->src, sg_nents_for_len(req->src, req->nbytes), + ctx->lastc, ctx->lastc_len, req->nbytes - ctx->lastc_len)) { + ret = -EINVAL; + goto exit; + } + + nbytes = ctx->hash_tmp_len + req->nbytes - ctx->lastc_len; + + /* not enough data */ + if (nbytes < RK_DMA_ALIGNMENT) { + CRYPTO_TRACE("nbytes = %u, not enough data", nbytes); + memcpy(ctx->hash_tmp + ctx->hash_tmp_len, + ctx->lastc, ctx->lastc_len); + ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len; + ctx->lastc_len = 0; + goto no_calc; + } + } else { + /* final just calc lastc_old */ + nbytes = ctx->hash_tmp_len; + CRYPTO_TRACE("nbytes = %u", nbytes); + } + + if (ctx->hash_tmp_len) { + /* Concatenate old data to the header */ + sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg)); + sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len); + sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src); + + src_sg = &ctx->hash_sg[0]; + ctx->hash_tmp_len = 0; + } else { + src_sg = req->src; + } + + alg_ctx->total = nbytes; + alg_ctx->left_bytes = nbytes; + alg_ctx->sg_src = src_sg; + alg_ctx->req_src = src_sg; + alg_ctx->src_nents = sg_nents_for_len(src_sg, nbytes); + + CRYPTO_TRACE("adjust: old_len = %u, new_len = %u, nbytes = %u", + ctx->hash_tmp_len, ctx->lastc_len, nbytes); + + if (nbytes) + ret = rk_ahash_set_data_start(rk_dev, rctx->is_final); + +exit: + return ret; +no_calc: + CRYPTO_TRACE("no calc"); + rk_alg_ctx_clear(alg_ctx); + + return 0; +} + +static int rk_ahash_get_zero_result(struct rk_crypto_dev *rk_dev, + uint8_t *data, uint32_t data_len) +{ + + struct ahash_request *req = + ahash_request_cast(rk_dev->async_req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - u32 reg_ctrl = 0; - CRYPTO_TRACE(); + return rk_ahash_fallback_digest(algt->name, algt->type == ALG_TYPE_HMAC, + ctx->authkey, ctx->authkey_len, + NULL, 0, + data, data_len); - alg_ctx->total = req->nbytes; - alg_ctx->left_bytes = req->nbytes; - alg_ctx->sg_src = req->src; - alg_ctx->req_src = req->src; - alg_ctx->src_nents = sg_nents_for_len(req->src, req->nbytes); - - if (algt->algo >= ARRAY_SIZE(hash_algo2bc)) - goto exit; - - reg_ctrl |= hash_algo2bc[algt->algo]; - clear_hash_out_reg(rk_dev); - - reg_ctrl |= CRYPTO_HW_PAD_ENABLE; - - if (IS_TYPE_HMAC(algt->type)) { - CRYPTO_TRACE("this is hmac"); - reg_ctrl |= CRYPTO_HMAC_ENABLE; - write_key_reg(rk_dev, ctx->authkey, sizeof(ctx->authkey)); - } - - CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, reg_ctrl | CRYPTO_WRITE_MASK_ALL); - - CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f); - - return rk_ahash_set_data_start(rk_dev); -exit: - CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, CRYPTO_WRITE_MASK_ALL | 0); - return -EINVAL; } - static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev, uint8_t *data, uint32_t data_len) { - int ret; + int ret = 0; u32 i, offset; u32 reg_ctrl = 0; + struct rk_hw_crypto_v2_info *hw_info = + (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; + + hw_info->is_started = false; + + /* use fallback hash */ + if (hw_info->hash_calc_cnt == 0) { + CRYPTO_TRACE("use fallback hash"); + ret = rk_ahash_get_zero_result(rk_dev, data, data_len); + goto exit; + } ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_HASH_VALID, reg_ctrl, @@ -465,6 +686,7 @@ static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev, CRYPTO_WRITE(rk_dev, CRYPTO_HASH_VALID, CRYPTO_HASH_IS_VALID); exit: + hw_info->hash_calc_cnt = 0; return ret; } @@ -473,9 +695,9 @@ static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) int err = 0; struct ahash_request *req = ahash_request_cast(rk_dev->async_req); struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - struct crypto_ahash *tfm; + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - CRYPTO_TRACE("left bytes = %u", alg_ctx->left_bytes); + CRYPTO_TRACE("left bytes = %u, is_final = %d", alg_ctx->left_bytes, rctx->is_final); err = rk_dev->unload_data(rk_dev); if (err) @@ -491,12 +713,21 @@ static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) } alg_ctx->sg_src = sg_next(alg_ctx->sg_src); } - err = rk_ahash_set_data_start(rk_dev); + err = rk_ahash_set_data_start(rk_dev, rctx->is_final); } else { /* * it will take some time to process date after last dma * transmission. */ + struct crypto_ahash *tfm; + + if (!rctx->is_final) + goto out_rx; + + if (!req->result) { + err = -EINVAL; + goto out_rx; + } tfm = crypto_ahash_reqtfm(req); @@ -525,30 +756,25 @@ static int rk_cra_hash_init(struct crypto_tfm *tfm) if (!rk_dev->request_crypto) return -EFAULT; - rk_dev->request_crypto(rk_dev, alg_name); - - alg_ctx->align_size = 64; + alg_ctx->align_size = RK_DMA_ALIGNMENT; alg_ctx->ops.start = rk_ahash_start; alg_ctx->ops.update = rk_ahash_crypto_rx; alg_ctx->ops.complete = rk_ahash_crypto_complete; alg_ctx->ops.irq_handle = rk_crypto_irq_handle; - ctx->rk_dev = rk_dev; - - /* for fallback */ - ctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->fallback_tfm)) { - dev_err(rk_dev->dev, "Could not load fallback driver.\n"); - return PTR_ERR(ctx->fallback_tfm); + ctx->rk_dev = rk_dev; + ctx->hash_tmp = (u8 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); + if (!ctx->hash_tmp) { + dev_err(rk_dev->dev, "Can't get zeroed page for hash tmp.\n"); + return -ENOMEM; } - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct rk_ahash_rctx) + - crypto_ahash_reqsize(ctx->fallback_tfm)); + rk_dev->request_crypto(rk_dev, alg_name); - algt->alg.hash.halg.statesize = crypto_ahash_statesize(ctx->fallback_tfm); + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct rk_ahash_rctx)); + + algt->alg.hash.halg.statesize = sizeof(struct rk_ahash_expt_ctx); return 0; } @@ -559,8 +785,10 @@ static void rk_cra_hash_exit(struct crypto_tfm *tfm) CRYPTO_TRACE(); - if (ctx->fallback_tfm) - crypto_free_ahash(ctx->fallback_tfm); + clean_hash_setting(ctx->rk_dev); + + if (ctx->hash_tmp) + free_page((unsigned long)ctx->hash_tmp); ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(tfm)); } diff --git a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c index 2cdde359e8ab..c3437077ab3a 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c @@ -9,6 +9,7 @@ * Some ideas are from marvell-cesa.c and s5p-sss.c driver. */ +#include #include #include @@ -220,7 +221,9 @@ static bool is_calc_need_round_up(struct skcipher_request *req) static void rk_cipher_reset(struct rk_crypto_dev *rk_dev) { + int ret; u32 tmp = 0, tmp_mask = 0; + unsigned int pool_timeout_us = 1000; CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00); @@ -228,8 +231,13 @@ static void rk_cipher_reset(struct rk_crypto_dev *rk_dev) tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT; CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask); - while (CRYPTO_READ(rk_dev, CRYPTO_RST_CTL)) - nop(); + + /* This is usually done in 20 clock cycles */ + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_RST_CTL, + tmp, !tmp, 0, pool_timeout_us); + if (ret) + dev_err(rk_dev->dev, "cipher reset pool timeout %ums.", + pool_timeout_us); CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0xffff0000); } @@ -255,7 +263,6 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err) pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl); pr_err("lli->usr_def = %08x\n", lli_desc->user_define); pr_err("lli->next = %08x\n\n\n", lli_desc->next_addr); - } if (base->complete) @@ -445,6 +452,8 @@ static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev) struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); u32 ivsize; + rk_cipher_reset(rk_dev); + CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000); ivsize = crypto_skcipher_ivsize(cipher);