diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 6fc61f506e7b..bf673a8e8854 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -749,11 +749,13 @@ config CRYPTO_DEV_IMGTEC_HASH config CRYPTO_DEV_ROCKCHIP tristate "Rockchip's Cryptographic Engine driver" depends on OF && ARCH_ROCKCHIP + select CRYPTO_DES select CRYPTO_AES select CRYPTO_ECB select CRYPTO_CBC select CRYPTO_XTS select CRYPTO_CFB + select CRYPTO_OFB select CRYPTO_CTR select CRYPTO_LIB_DES select CRYPTO_MD5 diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index 09171ffed83f..8a59317edebb 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -58,37 +58,6 @@ static struct rk_alg_ctx *rk_alg_ctx_cast(struct crypto_async_request *async_req return &ctx->algs_ctx; } -static void dump_alg_ctx(struct crypto_async_request *async_req) -{ - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(async_req); - struct scatterlist *cur_sg = NULL; - unsigned int i; - - CRYPTO_TRACE("\n"); - - CRYPTO_TRACE("------ req_src addr = %llx, nents = %zu ------", - (long long)alg_ctx->req_src, alg_ctx->src_nents); - - for_each_sg(alg_ctx->req_src, cur_sg, alg_ctx->src_nents, i) - CRYPTO_TRACE("sg %llx: virt = %llx, off = %u, len = %u", - (long long)cur_sg, (long long)sg_virt(cur_sg), - cur_sg->offset, cur_sg->length); - - CRYPTO_TRACE("\n"); - - if (!alg_ctx->req_dst) - return; - - CRYPTO_TRACE("------ req_dst addr = %llx, nents = %zu ------", - (long long)alg_ctx->req_dst, alg_ctx->dst_nents); - - for_each_sg(alg_ctx->req_dst, cur_sg, alg_ctx->dst_nents, i) - CRYPTO_TRACE("sg %llx: virt = %llx, off = %u, len = %u\n", - (long long)cur_sg, (long long)sg_virt(cur_sg), - cur_sg->offset, cur_sg->length); - CRYPTO_TRACE("\n"); -} - static int rk_crypto_enable_clk(struct rk_crypto_dev *rk_dev) { int ret; @@ -110,9 +79,9 @@ static void rk_crypto_disable_clk(struct rk_crypto_dev *rk_dev) clk_bulk_disable_unprepare(rk_dev->clks_num, rk_dev->clk_bulks); } -static int check_alignment(struct scatterlist *sg_src, - struct scatterlist *sg_dst, - int align_mask) +static int check_scatter_align(struct scatterlist *sg_src, + struct scatterlist *sg_dst, + int align_mask) { int in, out, align; @@ -127,6 +96,34 @@ static int check_alignment(struct scatterlist *sg_src, return (align && (sg_src->length == sg_dst->length)); } +static bool check_scatterlist_align(struct crypto_async_request *async_req, + int align_mask) +{ + struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(async_req); + struct scatterlist *src_tmp = NULL; + struct scatterlist *dst_tmp = NULL; + unsigned int i; + + if (alg_ctx->src_nents != alg_ctx->dst_nents) + return false; + + src_tmp = alg_ctx->req_src; + dst_tmp = alg_ctx->req_dst; + + for (i = 0; i < alg_ctx->src_nents; i++) { + if (!src_tmp || !dst_tmp) + return false; + + if (!check_scatter_align(src_tmp, dst_tmp, align_mask)) + return false; + + src_tmp = sg_next(src_tmp); + dst_tmp = sg_next(dst_tmp); + } + + return true; +} + static int rk_load_data(struct rk_crypto_dev *rk_dev, struct scatterlist *sg_src, struct scatterlist *sg_dst) @@ -136,9 +133,9 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, struct device *dev = rk_dev->dev; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); - mutex_lock(&rk_dev->mutex); - - alg_ctx->aligned = check_alignment(sg_src, sg_dst, alg_ctx->align_size); + if (alg_ctx->left_bytes == alg_ctx->total) + alg_ctx->aligned = check_scatterlist_align(rk_dev->async_req, + alg_ctx->align_size); CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n", alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes); @@ -208,7 +205,6 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, alg_ctx->count = count; return 0; error: - mutex_unlock(&rk_dev->mutex); return ret; } @@ -221,6 +217,9 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n", alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes); + if (alg_ctx->count == 0) + return 0; + sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp; dma_unmap_sg(rk_dev->dev, sg_in, 1, DMA_TO_DEVICE); @@ -240,10 +239,23 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) } exit: - mutex_unlock(&rk_dev->mutex); return ret; } +static void start_irq_timer(struct rk_crypto_dev *rk_dev) +{ + mod_timer(&rk_dev->timer, jiffies + msecs_to_jiffies(3000)); +} + +/* use timer to avoid crypto irq timeout */ +static void rk_crypto_irq_timer_handle(struct timer_list *t) +{ + struct rk_crypto_dev *rk_dev = from_timer(rk_dev, t, timer); + + rk_dev->err = -ETIMEDOUT; + tasklet_schedule(&rk_dev->done_task); +} + static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id) { struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id); @@ -260,6 +272,49 @@ static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id) return IRQ_HANDLED; } +static int rk_start_op(struct rk_crypto_dev *rk_dev) +{ + struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + int ret; + + if (!alg_ctx || !alg_ctx->ops.start) + return -EINVAL; + + alg_ctx->aligned = false; + + start_irq_timer(rk_dev); + + ret = alg_ctx->ops.start(rk_dev); + if (ret) + return ret; + + return 0; +} + +static int rk_update_op(struct rk_crypto_dev *rk_dev) +{ + struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + + if (!alg_ctx || !alg_ctx->ops.update) + return -EINVAL; + + return alg_ctx->ops.update(rk_dev); +} + +static void rk_complete_op(struct rk_crypto_dev *rk_dev, int err) +{ + struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + + del_timer(&rk_dev->timer); + + if (!alg_ctx || !alg_ctx->ops.complete) + return; + + alg_ctx->ops.complete(rk_dev->async_req, err); + + tasklet_schedule(&rk_dev->queue_task); +} + static int rk_crypto_enqueue(struct rk_crypto_dev *rk_dev, struct crypto_async_request *async_req) { @@ -283,9 +338,7 @@ static void rk_crypto_queue_task_cb(unsigned long data) { struct rk_crypto_dev *rk_dev = (struct rk_crypto_dev *)data; struct crypto_async_request *async_req, *backlog; - struct rk_alg_ctx *alg_ctx; unsigned long flags; - int err = 0; rk_dev->err = 0; spin_lock_irqsave(&rk_dev->lock, flags); @@ -304,14 +357,10 @@ static void rk_crypto_queue_task_cb(unsigned long data) backlog = NULL; } - alg_ctx = rk_alg_ctx_cast(async_req); - rk_dev->async_req = async_req; - err = alg_ctx->ops.start(rk_dev); - if (err) - alg_ctx->ops.complete(rk_dev->async_req, err); - - dump_alg_ctx(async_req); + rk_dev->err = rk_start_op(rk_dev); + if (rk_dev->err) + rk_complete_op(rk_dev, rk_dev->err); } static void rk_crypto_done_task_cb(unsigned long data) @@ -319,14 +368,19 @@ static void rk_crypto_done_task_cb(unsigned long data) struct rk_crypto_dev *rk_dev = (struct rk_crypto_dev *)data; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); - if (rk_dev->err) { - alg_ctx->ops.complete(rk_dev->async_req, rk_dev->err); - return; - } - - rk_dev->err = alg_ctx->ops.update(rk_dev); if (rk_dev->err) - alg_ctx->ops.complete(rk_dev->async_req, rk_dev->err); + goto exit; + + rk_dev->err = rk_update_op(rk_dev); + if (rk_dev->err) + goto exit; + + if (alg_ctx->total && alg_ctx->left_bytes == 0) + goto exit; + + return; +exit: + rk_complete_op(rk_dev, rk_dev->err); } static struct rk_crypto_algt *rk_crypto_find_algs(struct rk_crypto_dev *rk_dev, @@ -373,10 +427,18 @@ static int rk_crypto_register(struct rk_crypto_dev *rk_dev) tmp_algs->rk_dev = rk_dev; if (tmp_algs->type == ALG_TYPE_CIPHER) { - if (tmp_algs->algo == CIPHER_ALGO_AES && - tmp_algs->mode != CIPHER_MODE_XTS && - soc_data->use_soft_aes192) - tmp_algs->alg.crypto.base.cra_flags |= CRYPTO_ALG_NEED_FALLBACK; + if (tmp_algs->mode == CIPHER_MODE_CTR || + tmp_algs->mode == CIPHER_MODE_CFB || + tmp_algs->mode == CIPHER_MODE_OFB) + tmp_algs->alg.crypto.base.cra_blocksize = 1; + + if (tmp_algs->mode == CIPHER_MODE_ECB) + tmp_algs->alg.crypto.ivsize = 0; + + /* rv1126 is not support aes192 */ + if (soc_data->use_soft_aes192 && + tmp_algs->algo == CIPHER_ALGO_AES) + tmp_algs->use_soft_aes192 = true; err = crypto_register_skcipher(&tmp_algs->alg.crypto); } else if (tmp_algs->type == ALG_TYPE_HASH || tmp_algs->type == ALG_TYPE_HMAC) { @@ -685,7 +747,7 @@ static int rk_crypto_probe(struct platform_device *pdev) rk_crypto_done_task_cb, (unsigned long)rk_dev); crypto_init_queue(&rk_dev->queue, 50); - mutex_init(&rk_dev->mutex); + timer_setup(&rk_dev->timer, rk_crypto_irq_timer_handle, 0); rk_dev->request_crypto = rk_crypto_request; rk_dev->release_crypto = rk_crypto_release; @@ -704,7 +766,6 @@ static int rk_crypto_probe(struct platform_device *pdev) return 0; err_register_alg: - mutex_destroy(&rk_dev->mutex); tasklet_kill(&rk_dev->queue_task); tasklet_kill(&rk_dev->done_task); err_crypto: @@ -715,6 +776,8 @@ static int rk_crypto_remove(struct platform_device *pdev) { struct rk_crypto_dev *rk_dev = platform_get_drvdata(pdev); + del_timer_sync(&rk_dev->timer); + rk_crypto_unregister(rk_dev); tasklet_kill(&rk_dev->done_task); tasklet_kill(&rk_dev->queue_task); @@ -724,8 +787,6 @@ static int rk_crypto_remove(struct platform_device *pdev) rk_dev->soc_data->hw_deinit(&pdev->dev, rk_dev->hw_info); - mutex_destroy(&rk_dev->mutex); - return 0; } diff --git a/drivers/crypto/rockchip/rk_crypto_core.h b/drivers/crypto/rockchip/rk_crypto_core.h index 6a84117d1507..ee14ef10871a 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.h +++ b/drivers/crypto/rockchip/rk_crypto_core.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "rk_crypto_bignum.h" @@ -65,13 +66,13 @@ struct rk_crypto_dev { /* device lock */ spinlock_t lock; - struct mutex mutex; /* the public variable */ struct crypto_async_request *async_req; void *addr_vir; u32 vir_max; + struct timer_list timer; bool busy; void (*request_crypto)(struct rk_crypto_dev *rk_dev, const char *name); void (*release_crypto)(struct rk_crypto_dev *rk_dev, const char *name); @@ -107,8 +108,9 @@ struct rk_alg_ctx { dma_addr_t addr_in; dma_addr_t addr_out; - int aligned; + bool aligned; int align_size; + int chunk_size; }; /* the private variable of hash */ @@ -136,8 +138,10 @@ struct rk_cipher_ctx { unsigned int keylen; u32 mode; u8 iv[AES_BLOCK_SIZE]; + u8 lastc[AES_BLOCK_SIZE]; /* for fallback */ + bool fallback_key_inited; struct crypto_skcipher *fallback_tfm; struct skcipher_request fallback_req; // keep at the end }; @@ -169,6 +173,7 @@ struct rk_crypto_algt { u32 algo; u32 mode; char *name; + bool use_soft_aes192; }; enum rk_hash_algo { @@ -213,7 +218,8 @@ enum rk_cipher_mode { .base.cra_name = #algo_name,\ .base.cra_driver_name = #driver_name,\ .base.cra_priority = RK_CRYPTO_PRIORITY,\ - .base.cra_flags = CRYPTO_ALG_ASYNC,\ + .base.cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK,\ .base.cra_blocksize = cipher_algo##_BLOCK_SIZE,\ .base.cra_ctxsize = sizeof(struct rk_cipher_ctx),\ .base.cra_alignmask = 0x07,\ @@ -223,6 +229,7 @@ enum rk_cipher_mode { .min_keysize = cipher_algo##_MIN_KEY_SIZE,\ .max_keysize = cipher_algo##_MAX_KEY_SIZE,\ .ivsize = cipher_algo##_BLOCK_SIZE,\ + .chunksize = cipher_algo##_BLOCK_SIZE,\ .setkey = rk_cipher_setkey,\ .encrypt = rk_cipher_encrypt,\ .decrypt = rk_cipher_decrypt,\ @@ -238,7 +245,8 @@ enum rk_cipher_mode { .base.cra_name = #algo_name,\ .base.cra_driver_name = #driver_name,\ .base.cra_priority = RK_CRYPTO_PRIORITY,\ - .base.cra_flags = CRYPTO_ALG_ASYNC,\ + .base.cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK,\ .base.cra_blocksize = cipher_algo##_BLOCK_SIZE,\ .base.cra_ctxsize = sizeof(struct rk_cipher_ctx),\ .base.cra_alignmask = 0x07,\ @@ -248,6 +256,7 @@ enum rk_cipher_mode { .min_keysize = cipher_algo##_MAX_KEY_SIZE,\ .max_keysize = cipher_algo##_MAX_KEY_SIZE * 2,\ .ivsize = cipher_algo##_BLOCK_SIZE,\ + .chunksize = cipher_algo##_BLOCK_SIZE,\ .setkey = rk_cipher_setkey,\ .encrypt = rk_cipher_encrypt,\ .decrypt = rk_cipher_decrypt,\ diff --git a/drivers/crypto/rockchip/rk_crypto_v1_ahash.c b/drivers/crypto/rockchip/rk_crypto_v1_ahash.c index 2a5f4272d45c..e8f080f1a1c7 100644 --- a/drivers/crypto/rockchip/rk_crypto_v1_ahash.c +++ b/drivers/crypto/rockchip/rk_crypto_v1_ahash.c @@ -298,8 +298,6 @@ static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) tfm = crypto_ahash_reqtfm(req); memcpy_fromio(req->result, rk_dev->reg + RK_CRYPTO_HASH_DOUT_0, crypto_ahash_digestsize(tfm)); - alg_ctx->ops.complete(rk_dev->async_req, 0); - tasklet_schedule(&rk_dev->queue_task); } out_rx: diff --git a/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c index ed3be60bcdf2..8e0751bc3d83 100644 --- a/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c @@ -363,9 +363,6 @@ static int rk_ablk_rx(struct rk_crypto_dev *rk_dev) err = rk_set_data_start(rk_dev); } else { rk_iv_copyback(rk_dev); - /* here show the calculation is over without any err */ - alg_ctx->ops.complete(rk_dev->async_req, 0); - tasklet_schedule(&rk_dev->queue_task); } out_rx: return err; diff --git a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c index 47ce1fb6e90d..1b254c6e5a98 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c @@ -19,43 +19,6 @@ #define RK_HASH_CTX_MAGIC 0x1A1A1A1A #define RK_POLL_PERIOD_US 100 #define RK_POLL_TIMEOUT_US 50000 -#define HASH_MAX_SIZE PAGE_SIZE - -static const u8 null_hash_md5_value[] = { - 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, - 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e -}; - -static const u8 null_hash_sha1_value[] = { - 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, - 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, - 0xaf, 0xd8, 0x07, 0x09 -}; - -static const u8 null_hash_sha256_value[] = { - 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, - 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, - 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, - 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 -}; - -static const u8 null_hash_sha512_value[] = { - 0xcf, 0x83, 0xe1, 0x35, 0x7e, 0xef, 0xb8, 0xbd, - 0xf1, 0x54, 0x28, 0x50, 0xd6, 0x6d, 0x80, 0x07, - 0xd6, 0x20, 0xe4, 0x05, 0x0b, 0x57, 0x15, 0xdc, - 0x83, 0xf4, 0xa9, 0x21, 0xd3, 0x6c, 0xe9, 0xce, - 0x47, 0xd0, 0xd1, 0x3c, 0x5d, 0x85, 0xf2, 0xb0, - 0xff, 0x83, 0x18, 0xd2, 0x87, 0x7e, 0xec, 0x2f, - 0x63, 0xb9, 0x31, 0xbd, 0x47, 0x41, 0x7a, 0x81, - 0xa5, 0x38, 0x32, 0x7a, 0xf9, 0x27, 0xda, 0x3e -}; - -static const u8 null_hash_sm3_value[] = { - 0x1a, 0xb2, 0x1d, 0x83, 0x55, 0xcf, 0xa1, 0x7f, - 0x8e, 0x61, 0x19, 0x48, 0x31, 0xe8, 0x1a, 0x8f, - 0x22, 0xbe, 0xc8, 0xc7, 0x28, 0xfe, 0xfb, 0x74, - 0x7e, 0xd0, 0x35, 0xeb, 0x50, 0x82, 0xaa, 0x2b -}; static const u32 hash_algo2bc[] = { [HASH_ALGO_MD5] = CRYPTO_MD5, @@ -106,35 +69,6 @@ static struct rk_crypto_algt *rk_ahash_get_algt(struct crypto_ahash *tfm) return container_of(alg, struct rk_crypto_algt, alg.hash); } -static int zero_message_process(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - int rk_digest_size = crypto_ahash_digestsize(tfm); - struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); - - switch (algt->algo) { - case HASH_ALGO_MD5: - memcpy(req->result, null_hash_md5_value, rk_digest_size); - break; - case HASH_ALGO_SHA1: - memcpy(req->result, null_hash_sha1_value, rk_digest_size); - break; - case HASH_ALGO_SHA256: - memcpy(req->result, null_hash_sha256_value, rk_digest_size); - break; - case HASH_ALGO_SHA512: - memcpy(req->result, null_hash_sha512_value, rk_digest_size); - break; - case HASH_ALGO_SM3: - memcpy(req->result, null_hash_sm3_value, rk_digest_size); - break; - default: - return -EINVAL; - } - - return 0; -} - static int rk_crypto_irq_handle(int irq, void *dev_id) { struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id); @@ -179,6 +113,10 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) { + struct rk_ahash_ctx *ctx = crypto_tfm_ctx(base->tfm); + + CRYPTO_WRITE(ctx->rk_dev, CRYPTO_HASH_CTL, 0xffff0000); + if (base->complete) base->complete(base, err); } @@ -304,19 +242,25 @@ static int rk_ahash_export(struct ahash_request *req, void *out) static int rk_ahash_digest(struct ahash_request *req) { - struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); - struct rk_crypto_dev *rk_dev = tctx->rk_dev; + struct rk_ahash_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_dev *rk_dev = ctx->rk_dev; CRYPTO_TRACE("calc data %u bytes.", req->nbytes); - if (!req->nbytes) - return IS_TYPE_HMAC(algt->type) ? - crypto_ahash_digest(req) : - zero_message_process(req); - else + if (!req->nbytes || req->nbytes > rk_dev->vir_max) { + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + rctx->fallback_req.result = req->result; + + return crypto_ahash_digest(&rctx->fallback_req); + } else { return rk_dev->enqueue(rk_dev, &req->base); + } } static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen, @@ -484,7 +428,7 @@ static int rk_ahash_start(struct rk_crypto_dev *rk_dev) return rk_ahash_set_data_start(rk_dev); exit: CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, CRYPTO_WRITE_MASK_ALL | 0); - return -1; + return -EINVAL; } static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev, @@ -554,9 +498,6 @@ static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) err = rk_ahash_get_result(rk_dev, req->result, crypto_ahash_digestsize(tfm)); - - alg_ctx->ops.complete(rk_dev->async_req, err); - tasklet_schedule(&rk_dev->queue_task); } out_rx: @@ -582,7 +523,7 @@ static int rk_cra_hash_init(struct crypto_tfm *tfm) rk_dev->request_crypto(rk_dev, alg_name); - alg_ctx->align_size = 4; + alg_ctx->align_size = 64; alg_ctx->ops.start = rk_ahash_start; alg_ctx->ops.update = rk_ahash_crypto_rx; diff --git a/drivers/crypto/rockchip/rk_crypto_v2_reg.h b/drivers/crypto/rockchip/rk_crypto_v2_reg.h index c2a0ef10e85e..d149265dfab6 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_reg.h +++ b/drivers/crypto/rockchip/rk_crypto_v2_reg.h @@ -243,6 +243,19 @@ #define CRYPTO_HASH_VALID 0x03e4 #define CRYPTO_HASH_IS_VALID BIT(0) +#define LLI_DMA_CTRL_LAST BIT(0) +#define LLI_DMA_CTRL_PAUSE BIT(1) +#define LLI_DMA_CTRL_LIST_DONE BIT(8) +#define LLI_DMA_CTRL_DST_DONE BIT(9) +#define LLI_DMA_CTRL_SRC_DONE BIT(10) + +#define LLI_USER_CIPHER_START BIT(0) +#define LLI_USER_STRING_START BIT(1) +#define LLI_USER_STRING_LAST BIT(2) +#define LLI_USER_STRING_AAD BIT(3) +#define LLI_USER_PRIVACY_KEY BIT(7) +#define LLI_USER_ROOT_KEY BIT(8) + #define CRYPTO_PKA_BASE_OFFSET 0x0480 #define CRYPTO_RAM_CTL (0x0480 - CRYPTO_PKA_BASE_OFFSET) diff --git a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c index 7cca1ba7a1c0..d940f1dc8949 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c @@ -166,9 +166,19 @@ static struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm) return container_of(alg, struct rk_crypto_algt, alg.crypto); } -static bool is_use_fallback(struct rk_cipher_ctx *ctx) +static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len) { - return ctx->keylen == AES_KEYSIZE_192 && ctx->fallback_tfm; + if (algt->algo != CIPHER_ALGO_AES) + return false; + + /* crypto v2 not support xts with AES-192 */ + if (algt->mode == CIPHER_MODE_XTS && key_len == AES_KEYSIZE_192 * 2) + return true; + + if (algt->use_soft_aes192 && key_len == AES_KEYSIZE_192) + return true; + + return false; } static bool is_no_multi_blocksize(struct skcipher_request *req) @@ -176,13 +186,62 @@ static bool is_no_multi_blocksize(struct skcipher_request *req) struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + return (algt->mode == CIPHER_MODE_CFB || + algt->mode == CIPHER_MODE_OFB || + algt->mode == CIPHER_MODE_CTR || + algt->mode == CIPHER_MODE_XTS) ? true : false; +} + +static bool is_calc_need_round_up(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + return (algt->mode == CIPHER_MODE_CFB || algt->mode == CIPHER_MODE_OFB || algt->mode == CIPHER_MODE_CTR) ? true : false; } +static void rk_cipher_reset(struct rk_crypto_dev *rk_dev) +{ + u32 tmp = 0, tmp_mask = 0; + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00); + + tmp = CRYPTO_SW_CC_RESET; + tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT; + + CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask); + while (CRYPTO_READ(rk_dev, CRYPTO_RST_CTL)) + nop(); + + CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0xffff0000); +} + static void rk_crypto_complete(struct crypto_async_request *base, int err) { + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info; + struct crypto_lli_desc *lli_desc = hw_info->desc; + + CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000); + if (err) { + rk_cipher_reset(ctx->rk_dev); + pr_err("aligned = %u, align_size = %u\n", + alg_ctx->aligned, alg_ctx->align_size); + pr_err("total = %u, left = %u, count = %u\n", + alg_ctx->total, alg_ctx->left_bytes, alg_ctx->count); + pr_err("lli->src = %08x\n", lli_desc->src_addr); + pr_err("lli->src_len = %08x\n", lli_desc->src_len); + pr_err("lli->dst = %08x\n", lli_desc->dst_addr); + pr_err("lli->dst_len = %08x\n", lli_desc->dst_len); + pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl); + pr_err("lli->usr_def = %08x\n", lli_desc->user_define); + pr_err("lli->next = %08x\n\n\n", lli_desc->next_addr); + + } + if (base->complete) base->complete(base, err); } @@ -192,7 +251,7 @@ static int rk_handle_req(struct rk_crypto_dev *rk_dev, { struct rk_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - if (!IS_ALIGNED(req->cryptlen, ctx->algs_ctx.align_size) && + if (!IS_ALIGNED(req->cryptlen, ctx->algs_ctx.chunk_size) && !is_no_multi_blocksize(req)) return -EINVAL; else @@ -204,65 +263,47 @@ static int rk_cipher_setkey(struct crypto_skcipher *cipher, { struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + uint32_t key_factor; int ret = -EINVAL; CRYPTO_MSG("algo = %x, mode = %x, key_len = %d\n", algt->algo, algt->mode, keylen); + /* The key length of XTS is twice the normal length */ + key_factor = algt->mode == CIPHER_MODE_XTS ? 2 : 1; + switch (algt->algo) { case CIPHER_ALGO_DES: ret = verify_skcipher_des_key(cipher, key); if (ret) - goto error; + goto exit; break; case CIPHER_ALGO_DES3_EDE: ret = verify_skcipher_des3_key(cipher, key); if (ret) - goto error; + goto exit; break; case CIPHER_ALGO_AES: - if (algt->mode != CIPHER_MODE_XTS) { - if (keylen != AES_KEYSIZE_128 && - keylen != AES_KEYSIZE_192 && - keylen != AES_KEYSIZE_256) - goto error; - } else { - if (keylen != AES_KEYSIZE_256 && - keylen != AES_KEYSIZE_256 * 2) - goto error; - } + if (keylen != (AES_KEYSIZE_128 * key_factor) && + keylen != (AES_KEYSIZE_192 * key_factor) && + keylen != (AES_KEYSIZE_256 * key_factor)) + goto exit; break; case CIPHER_ALGO_SM4: - if (algt->mode != CIPHER_MODE_XTS) { - if (keylen != SM4_KEY_SIZE) - goto error; - } else { - if (keylen != SM4_KEY_SIZE * 2) - goto error; - } + if (keylen != (SM4_KEY_SIZE * key_factor)) + goto exit; break; default: - goto error; + ret = -EINVAL; + goto exit; } memcpy(ctx->key, key, keylen); ctx->keylen = keylen; + ctx->fallback_key_inited = false; - if (algt->mode == CIPHER_MODE_XTS) - ctx->keylen /= 2; - - if (is_use_fallback(ctx)) { - CRYPTO_MSG("use fallback tfm"); - ret = crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); - if (ret) { - CRYPTO_MSG("soft fallback crypto_skcipher_setkey err = %d\n", ret); - goto error; - } - } - - return 0; - -error: + ret = 0; +exit: return ret; } @@ -274,6 +315,24 @@ static int rk_cipher_fallback(struct skcipher_request *req, CRYPTO_MSG("use fallback tfm"); + if (!ctx->fallback_tfm) { + ret = -ENODEV; + CRYPTO_MSG("fallback_tfm is empty!\n"); + goto exit; + } + + if (!ctx->fallback_key_inited) { + ret = crypto_skcipher_setkey(ctx->fallback_tfm, + ctx->key, ctx->keylen); + if (ret) { + CRYPTO_MSG("fallback crypto_skcipher_setkey err = %d\n", + ret); + goto exit; + } + + ctx->fallback_key_inited = true; + } + skcipher_request_set_tfm(&ctx->fallback_req, ctx->fallback_tfm); skcipher_request_set_callback(&ctx->fallback_req, req->base.flags, @@ -286,6 +345,7 @@ static int rk_cipher_fallback(struct skcipher_request *req, ret = encrypt ? crypto_skcipher_encrypt(&ctx->fallback_req) : crypto_skcipher_decrypt(&ctx->fallback_req); +exit: return ret; } @@ -294,24 +354,61 @@ static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt) struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); struct rk_crypto_algt *algt = rk_cipher_get_algt(tfm); - int ret = -EINVAL; CRYPTO_TRACE("%s total = %u", encrypt ? "encrypt" : "decrypt", req->cryptlen); - if (is_use_fallback(ctx)) { - ret = rk_cipher_fallback(req, ctx, encrypt); - } else { - ctx->mode = cipher_algo2bc[algt->algo] | - cipher_mode2bc[algt->mode]; - if (!encrypt) - ctx->mode |= CRYPTO_BC_DECRYPT; - - CRYPTO_MSG("ctx->mode = %x\n", ctx->mode); - ret = rk_handle_req(ctx->rk_dev, req); + if (!req->cryptlen) { + if (algt->mode == CIPHER_MODE_ECB || + algt->mode == CIPHER_MODE_CBC || + algt->mode == CIPHER_MODE_CTR || + algt->mode == CIPHER_MODE_CFB || + algt->mode == CIPHER_MODE_OFB) + return 0; + else + return -EINVAL; } - return ret; + /* XTS data should >= chunksize */ + if (algt->mode == CIPHER_MODE_XTS && + req->cryptlen < crypto_skcipher_chunksize(tfm)) + return -EINVAL; + + if (is_force_fallback(algt, ctx->keylen) || + req->cryptlen > ctx->rk_dev->vir_max) { + return rk_cipher_fallback(req, ctx, encrypt); + } + + ctx->mode = cipher_algo2bc[algt->algo] | + cipher_mode2bc[algt->mode]; + if (!encrypt) + ctx->mode |= CRYPTO_BC_DECRYPT; + + if (algt->algo == CIPHER_ALGO_AES) { + uint32_t key_factor; + + /* The key length of XTS is twice the normal length */ + key_factor = algt->mode == CIPHER_MODE_XTS ? 2 : 1; + + if (ctx->keylen == AES_KEYSIZE_128 * key_factor) + ctx->mode |= CRYPTO_BC_128_bit_key; + else if (ctx->keylen == AES_KEYSIZE_192 * key_factor) + ctx->mode |= CRYPTO_BC_192_bit_key; + else if (ctx->keylen == AES_KEYSIZE_256 * key_factor) + ctx->mode |= CRYPTO_BC_256_bit_key; + } + + if (!encrypt && (req->src == req->dst)) { + u32 ivsize = crypto_skcipher_ivsize(tfm); + + if (req->cryptlen >= ivsize) + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + ctx->lastc, ivsize, + req->cryptlen - ivsize); + } + + CRYPTO_MSG("ctx->mode = %x\n", ctx->mode); + return rk_handle_req(ctx->rk_dev, req); } static int rk_cipher_encrypt(struct skcipher_request *req) @@ -329,32 +426,25 @@ static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev) struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); - u32 ivsize, block; + u32 ivsize; CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000); - block = crypto_tfm_alg_blocksize(tfm); ivsize = crypto_skcipher_ivsize(cipher); - write_key_reg(ctx->rk_dev, ctx->key, ctx->keylen); - if (MASK_BC_MODE(ctx->mode) == CRYPTO_BC_XTS) - write_tkey_reg(ctx->rk_dev, - ctx->key + ctx->keylen, ctx->keylen); + if (MASK_BC_MODE(ctx->mode) == CRYPTO_BC_XTS) { + uint32_t tmp_len = ctx->keylen / 2; + + write_key_reg(ctx->rk_dev, ctx->key, tmp_len); + write_tkey_reg(ctx->rk_dev, ctx->key + tmp_len, tmp_len); + } else { + write_key_reg(ctx->rk_dev, ctx->key, ctx->keylen); + } if (MASK_BC_MODE(ctx->mode) != CRYPTO_BC_ECB) set_iv_reg(rk_dev, req->iv, ivsize); - if (block != DES_BLOCK_SIZE) { - if (ctx->keylen == AES_KEYSIZE_128) - ctx->mode |= CRYPTO_BC_128_bit_key; - else if (ctx->keylen == AES_KEYSIZE_192) - ctx->mode |= CRYPTO_BC_192_bit_key; - else if (ctx->keylen == AES_KEYSIZE_256) - ctx->mode |= CRYPTO_BC_256_bit_key; - } - ctx->mode |= CRYPTO_BC_ENABLE; CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003); @@ -372,28 +462,31 @@ static void crypto_dma_start(struct rk_crypto_dev *rk_dev) skcipher_request_cast(rk_dev->async_req); struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); u32 calc_len = alg_ctx->count; + u32 start_flag = CRYPTO_DMA_START; memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); /* * the data length is not aligned will use addr_vir to calculate, - * so crypto v2 could round up date date length to align_size + * so crypto v2 could round up data length to chunk_size */ - if (is_no_multi_blocksize(req)) - calc_len = round_up(calc_len, alg_ctx->align_size); + if (is_calc_need_round_up(req)) + calc_len = round_up(calc_len, alg_ctx->chunk_size); hw_info->desc->src_addr = alg_ctx->addr_in; hw_info->desc->src_len = calc_len; hw_info->desc->dst_addr = alg_ctx->addr_out; hw_info->desc->dst_len = calc_len; hw_info->desc->next_addr = 0; - hw_info->desc->dma_ctrl = 0x00000201; - hw_info->desc->user_define = 0x7; + hw_info->desc->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; + hw_info->desc->user_define = LLI_USER_STRING_START | + LLI_USER_CIPHER_START | + LLI_USER_STRING_LAST; dma_wmb(); CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, 0x00010001);/* start */ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); } static int rk_set_data_start(struct rk_crypto_dev *rk_dev) @@ -433,29 +526,65 @@ static int rk_ablk_start(struct rk_crypto_dev *rk_dev) return err; } +/* increment counter (128-bit int) by 1 */ +static void rk_ctr128_inc(uint8_t *counter) +{ + u32 n = 16; + u8 c; + + do { + --n; + c = counter[n]; + ++c; + counter[n] = c; + if (c) + return; + } while (n); +} + +static void rk_ctr128_calc(uint8_t *counter, uint32_t data_len) +{ + u32 i; + u32 chunksize = AES_BLOCK_SIZE; + + for (i = 0; i < DIV_ROUND_UP(data_len, chunksize); i++) + rk_ctr128_inc(counter); +} + static void rk_iv_copyback(struct rk_crypto_dev *rk_dev) { struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - u32 ivsize = crypto_skcipher_ivsize(tfm); - /* Update the IV buffer to contain the next IV for encryption mode. */ - if (!IS_BC_DECRYPT(ctx->mode) && req->iv) { - if (alg_ctx->aligned) { - memcpy(req->iv, sg_virt(alg_ctx->sg_dst) + - alg_ctx->count - ivsize, ivsize); + u32 bc_mode = MASK_BC_MODE(ctx->mode); + + if (!req->iv) + return; + + if (bc_mode == CRYPTO_BC_CTR) { + /* calc new counter for CTR mode */ + rk_ctr128_calc(req->iv, req->cryptlen); + } else if (bc_mode == CRYPTO_BC_CBC) { + if (!IS_BC_DECRYPT(ctx->mode)) { + sg_pcopy_to_buffer(req->dst, sg_nents(req->dst), + req->iv, ivsize, + req->cryptlen - ivsize); } else { - memcpy(req->iv, rk_dev->addr_vir + - alg_ctx->count - ivsize, ivsize); + if (req->src == req->dst) + memcpy(req->iv, ctx->lastc, ivsize); + else + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + req->iv, ivsize, + req->cryptlen - ivsize); } + + } else { + /* do nothing */ } - } - /* return: * true some err was occurred * fault no err, continue @@ -485,9 +614,6 @@ static int rk_ablk_rx(struct rk_crypto_dev *rk_dev) err = rk_set_data_start(rk_dev); } else { rk_iv_copyback(rk_dev); - /* here show the calculation is over without any err */ - alg_ctx->ops.complete(rk_dev->async_req, 0); - tasklet_schedule(&rk_dev->queue_task); } out_rx: return err; @@ -510,7 +636,9 @@ static int rk_ablk_init_tfm(struct crypto_skcipher *tfm) rk_dev->request_crypto(rk_dev, alg_name); - alg_ctx->align_size = crypto_skcipher_blocksize(tfm); + /* always not aligned for crypto v2 cipher */ + alg_ctx->align_size = rk_dev->vir_max; + alg_ctx->chunk_size = crypto_skcipher_chunksize(tfm); alg_ctx->ops.start = rk_ablk_start; alg_ctx->ops.update = rk_ablk_rx; @@ -525,9 +653,9 @@ static int rk_ablk_init_tfm(struct crypto_skcipher *tfm) CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->fallback_tfm)) { - dev_err(rk_dev->dev, "Could not load fallback driver %s : %ld.\n", - alg_name, PTR_ERR(ctx->fallback_tfm)); - return PTR_ERR(ctx->fallback_tfm); + CRYPTO_MSG("Could not load fallback driver %s : %ld.\n", + alg_name, PTR_ERR(ctx->fallback_tfm)); + ctx->fallback_tfm = NULL; } }