From b71bd8763dab67e728cc94d49ec160ad7c7dbec8 Mon Sep 17 00:00:00 2001 From: Lin Jinhan Date: Tue, 12 Apr 2022 17:08:59 +0800 Subject: [PATCH] crypto: rockchip: v2&v3 add support gcm Signed-off-by: Lin Jinhan Change-Id: I7e63440c6f652b8939ade56d51957279310bec51 --- drivers/crypto/Kconfig | 1 + drivers/crypto/rockchip/rk_crypto_core.c | 92 ++++++- drivers/crypto/rockchip/rk_crypto_core.h | 43 +++ .../rockchip/rk_crypto_skcipher_utils.c | 191 ++++++++++++- .../rockchip/rk_crypto_skcipher_utils.h | 17 ++ drivers/crypto/rockchip/rk_crypto_utils.c | 25 +- drivers/crypto/rockchip/rk_crypto_utils.h | 2 + drivers/crypto/rockchip/rk_crypto_v2.c | 2 + drivers/crypto/rockchip/rk_crypto_v2.h | 2 + drivers/crypto/rockchip/rk_crypto_v2_reg.h | 42 +++ .../crypto/rockchip/rk_crypto_v2_skcipher.c | 252 +++++++++++++++-- drivers/crypto/rockchip/rk_crypto_v3.c | 72 ++--- drivers/crypto/rockchip/rk_crypto_v3.h | 2 + .../crypto/rockchip/rk_crypto_v3_skcipher.c | 256 ++++++++++++++++-- 14 files changed, 902 insertions(+), 97 deletions(-) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index e06b6465eeb3..4aed2f445576 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -757,6 +757,7 @@ config CRYPTO_DEV_ROCKCHIP select CRYPTO_CFB select CRYPTO_OFB select CRYPTO_CTR + select CRYPTO_GCM select CRYPTO_LIB_DES select CRYPTO_MD5 select CRYPTO_SHA1 diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index b63d131a09c1..93b8228d7327 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -10,6 +10,7 @@ * Some ideas are from marvell-cesa.c and s5p-sss.c driver. */ +#include #include #include #include @@ -60,6 +61,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, { int ret = -EINVAL; unsigned int count; + u32 src_nents, dst_nents; struct device *dev = rk_dev->dev; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); @@ -69,13 +71,58 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, if (alg_ctx->total == 0) return 0; + src_nents = alg_ctx->src_nents; + dst_nents = alg_ctx->dst_nents; + + /* skip assoclen data */ + if (alg_ctx->assoclen && alg_ctx->left_bytes == alg_ctx->total) { + CRYPTO_TRACE("have assoclen..."); + + if (alg_ctx->assoclen > rk_dev->aad_max) { + ret = -ENOMEM; + goto error; + } + + if (!sg_pcopy_to_buffer(alg_ctx->req_src, alg_ctx->src_nents, + rk_dev->addr_aad, alg_ctx->assoclen, 0)) { + dev_err(dev, "[%s:%d] assoc pcopy err\n", + __func__, __LINE__); + ret = -EINVAL; + goto error; + } + + sg_init_one(&alg_ctx->sg_aad, rk_dev->addr_aad, alg_ctx->assoclen); + + if (!dma_map_sg(dev, &alg_ctx->sg_aad, 1, DMA_TO_DEVICE)) { + dev_err(dev, "[%s:%d] dma_map_sg(sg_aad) error\n", + __func__, __LINE__); + ret = -ENOMEM; + goto error; + } + + alg_ctx->addr_aad_in = sg_dma_address(&alg_ctx->sg_aad); + + /* point sg_src and sg_dst skip assoc data */ + sg_src = scatterwalk_ffwd(rk_dev->src, alg_ctx->req_src, + alg_ctx->assoclen); + sg_dst = (alg_ctx->req_src == alg_ctx->req_dst) ? sg_src : + scatterwalk_ffwd(rk_dev->dst, alg_ctx->req_dst, + alg_ctx->assoclen); + + alg_ctx->sg_src = sg_src; + alg_ctx->sg_dst = sg_dst; + src_nents = sg_nents_for_len(sg_src, alg_ctx->total); + dst_nents = sg_nents_for_len(sg_dst, alg_ctx->total); + + CRYPTO_TRACE("src_nents = %u, dst_nents = %u", src_nents, dst_nents); + } + if (alg_ctx->left_bytes == alg_ctx->total) - alg_ctx->aligned = rk_crypto_check_align(alg_ctx->req_src, alg_ctx->src_nents, - alg_ctx->req_dst, alg_ctx->dst_nents, + alg_ctx->aligned = rk_crypto_check_align(sg_src, src_nents, sg_dst, dst_nents, alg_ctx->align_size); - CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n", - alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes); + CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u, assoclen = %u\n", + alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes, alg_ctx->assoclen); if (alg_ctx->aligned) { u32 nents; @@ -118,7 +165,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, if (!sg_pcopy_to_buffer(alg_ctx->req_src, alg_ctx->src_nents, rk_dev->addr_vir, count, - alg_ctx->total - alg_ctx->left_bytes)) { + alg_ctx->assoclen + alg_ctx->total - alg_ctx->left_bytes)) { dev_err(dev, "[%s:%d] pcopy err\n", __func__, __LINE__); ret = -EINVAL; @@ -183,12 +230,22 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) if (!sg_pcopy_from_buffer(alg_ctx->req_dst, alg_ctx->dst_nents, rk_dev->addr_vir, alg_ctx->count, alg_ctx->total - alg_ctx->left_bytes - - alg_ctx->count)) { + alg_ctx->count + alg_ctx->assoclen)) { ret = -EINVAL; goto exit; } } + if (alg_ctx->assoclen) { + dma_unmap_sg(rk_dev->dev, &alg_ctx->sg_aad, 1, DMA_TO_DEVICE); + + /* copy assoc data to dst */ + if (!sg_pcopy_from_buffer(alg_ctx->req_dst, sg_nents(alg_ctx->req_dst), + rk_dev->addr_aad, alg_ctx->assoclen, 0)) { + ret = -EINVAL; + goto exit; + } + } exit: return ret; } @@ -421,6 +478,11 @@ static int rk_crypto_register(struct rk_crypto_dev *rk_dev) err = crypto_register_ahash(&tmp_algs->alg.hash); } else if (tmp_algs->type == ALG_TYPE_ASYM) { err = crypto_register_akcipher(&tmp_algs->alg.asym); + } else if (tmp_algs->type == ALG_TYPE_AEAD) { + if (soc_data->use_soft_aes192 && + tmp_algs->algo == CIPHER_ALGO_AES) + tmp_algs->use_soft_aes192 = true; + err = crypto_register_aead(&tmp_algs->alg.aead); } else { continue; } @@ -449,6 +511,8 @@ err_cipher_algs: crypto_unregister_ahash(&tmp_algs->alg.hash); else if (tmp_algs->type == ALG_TYPE_ASYM) crypto_unregister_akcipher(&tmp_algs->alg.asym); + else if (tmp_algs->type == ALG_TYPE_AEAD) + crypto_unregister_aead(&tmp_algs->alg.aead); } rk_dev->release_crypto(rk_dev, __func__); @@ -524,8 +588,8 @@ static char *crypto_rv1126_algs_name[] = { }; static char *crypto_full_algs_name[] = { - "ecb(sm4)", "cbc(sm4)", "cfb(sm4)", "ofb(sm4)", "ctr(sm4)", - "ecb(aes)", "cbc(aes)", "cfb(aes)", "ofb(aes)", "ctr(aes)", + "ecb(sm4)", "cbc(sm4)", "cfb(sm4)", "ofb(sm4)", "ctr(sm4)", "gcm(sm4)", + "ecb(aes)", "cbc(aes)", "cfb(aes)", "ofb(aes)", "ctr(aes)", "gcm(aes)", "ecb(des)", "cbc(des)", "cfb(des)", "ofb(des)", "ecb(des3_ede)", "cbc(des3_ede)", "cfb(des3_ede)", "ofb(des3_ede)", "sha1", "sha224", "sha256", "sha384", "sha512", "md5", "sm3", @@ -710,6 +774,15 @@ static int rk_crypto_probe(struct platform_device *pdev) rk_dev->vir_max = RK_BUFFER_SIZE; + rk_dev->addr_aad = (void *)__get_free_page(GFP_KERNEL); + if (!rk_dev->addr_aad) { + err = -ENOMEM; + dev_err(dev, "__get_free_page failed.\n"); + goto err_crypto; + } + + rk_dev->aad_max = RK_BUFFER_SIZE; + platform_set_drvdata(pdev, rk_dev); tasklet_init(&rk_dev->queue_task, @@ -760,6 +833,9 @@ static int rk_crypto_remove(struct platform_device *pdev) if (rk_dev->addr_vir) free_pages((unsigned long)rk_dev->addr_vir, RK_BUFFER_ORDER); + if (rk_dev->addr_aad) + free_page((unsigned long)rk_dev->addr_aad); + rk_dev->soc_data->hw_deinit(&pdev->dev, rk_dev->hw_info); return 0; diff --git a/drivers/crypto/rockchip/rk_crypto_core.h b/drivers/crypto/rockchip/rk_crypto_core.h index 2d2f558a6e1b..ba85cc6c1b47 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.h +++ b/drivers/crypto/rockchip/rk_crypto_core.h @@ -12,7 +12,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -68,6 +70,10 @@ struct rk_crypto_dev { struct crypto_async_request *async_req; void *addr_vir; u32 vir_max; + void *addr_aad; + int aad_max; + struct scatterlist src[2]; + struct scatterlist dst[2]; struct timer_list timer; bool busy; @@ -116,18 +122,22 @@ struct rk_alg_ctx { struct scatterlist *sg_src; struct scatterlist *sg_dst; struct scatterlist sg_tmp; + struct scatterlist sg_aad; struct scatterlist *req_src; struct scatterlist *req_dst; size_t src_nents; size_t dst_nents; size_t map_nents; + int is_aead; unsigned int total; + unsigned int assoclen; unsigned int count; unsigned int left_bytes; dma_addr_t addr_in; dma_addr_t addr_out; + dma_addr_t addr_aad_in; bool aligned; int align_size; @@ -169,6 +179,7 @@ struct rk_cipher_ctx { unsigned int keylen; u32 mode; u8 iv[AES_BLOCK_SIZE]; + u32 iv_len; u8 lastc[AES_BLOCK_SIZE]; bool is_enc; void *priv; @@ -177,6 +188,7 @@ struct rk_cipher_ctx { bool fallback_key_inited; struct crypto_skcipher *fallback_tfm; struct skcipher_request fallback_req; // keep at the end + struct crypto_aead *fallback_aead; }; struct rk_rsa_ctx { @@ -193,6 +205,7 @@ enum alg_type { ALG_TYPE_HMAC, ALG_TYPE_CIPHER, ALG_TYPE_ASYM, + ALG_TYPE_AEAD, }; struct rk_crypto_algt { @@ -201,6 +214,7 @@ struct rk_crypto_algt { struct skcipher_alg crypto; struct ahash_alg hash; struct akcipher_alg asym; + struct aead_alg aead; } alg; enum alg_type type; u32 algo; @@ -251,6 +265,35 @@ enum rk_cipher_mode { #define MD5_BLOCK_SIZE SHA1_BLOCK_SIZE +#define RK_AEAD_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\ + .name = #algo_name,\ + .type = ALG_TYPE_AEAD,\ + .algo = CIPHER_ALGO_##cipher_algo,\ + .mode = CIPHER_MODE_##cipher_mode,\ + .alg.aead = {\ + .base.cra_name = #algo_name,\ + .base.cra_driver_name = #driver_name,\ + .base.cra_priority = RK_CRYPTO_PRIORITY,\ + .base.cra_flags = CRYPTO_ALG_TYPE_AEAD |\ + CRYPTO_ALG_KERN_DRIVER_ONLY |\ + CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK,\ + .base.cra_blocksize = 1,\ + .base.cra_ctxsize = sizeof(struct rk_cipher_ctx),\ + .base.cra_alignmask = 0x07,\ + .base.cra_module = THIS_MODULE,\ + .init = rk_aead_init_tfm,\ + .exit = rk_aead_exit_tfm,\ + .ivsize = GCM_AES_IV_SIZE,\ + .chunksize = cipher_algo##_BLOCK_SIZE,\ + .maxauthsize = AES_BLOCK_SIZE,\ + .setkey = rk_aead_setkey,\ + .setauthsize = rk_aead_gcm_setauthsize,\ + .encrypt = rk_aead_encrypt,\ + .decrypt = rk_aead_decrypt,\ + } \ +} + #define RK_CIPHER_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\ .name = #algo_name,\ .type = ALG_TYPE_CIPHER,\ diff --git a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c index 15f51b04bd10..27b4164e3085 100644 --- a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c +++ b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c @@ -17,11 +17,16 @@ struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm) return container_of(alg, struct rk_crypto_algt, alg.crypto); } -static struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev) +struct rk_crypto_algt *rk_aead_get_algt(struct crypto_aead *tfm) { - struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aead_alg *alg = crypto_aead_alg(tfm); + + return container_of(alg, struct rk_crypto_algt, alg.aead); +} + +struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(rk_dev->async_req->tfm); return ctx; } @@ -31,15 +36,13 @@ struct rk_alg_ctx *rk_cipher_alg_ctx(struct rk_crypto_dev *rk_dev) return &(rk_cipher_ctx_cast(rk_dev)->algs_ctx); } -static bool is_no_multi_blocksize(struct skcipher_request *req) +static bool is_no_multi_blocksize(uint32_t mode) { - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); - - return (algt->mode == CIPHER_MODE_CFB || - algt->mode == CIPHER_MODE_OFB || - algt->mode == CIPHER_MODE_CTR || - algt->mode == CIPHER_MODE_XTS) ? true : false; + return (mode == CIPHER_MODE_CFB || + mode == CIPHER_MODE_OFB || + mode == CIPHER_MODE_CTR || + mode == CIPHER_MODE_XTS || + mode == CIPHER_MODE_GCM) ? true : false; } int rk_cipher_fallback(struct skcipher_request *req, struct rk_cipher_ctx *ctx, bool encrypt) @@ -251,6 +254,7 @@ exit: int rk_ablk_rx(struct rk_crypto_dev *rk_dev) { int err = 0; + struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); CRYPTO_TRACE("left_bytes = %u\n", alg_ctx->left_bytes); @@ -273,7 +277,48 @@ int rk_ablk_rx(struct rk_crypto_dev *rk_dev) } err = rk_set_data_start(rk_dev); } else { - rk_iv_copyback(rk_dev); + if (alg_ctx->is_aead) { + u8 hard_tag[RK_MAX_TAG_SIZE]; + u8 user_tag[RK_MAX_TAG_SIZE]; + struct aead_request *req = + aead_request_cast(rk_dev->async_req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + unsigned int authsize = crypto_aead_authsize(tfm); + + CRYPTO_TRACE("cryptlen = %u, assoclen = %u, aead authsize = %u", + alg_ctx->total, alg_ctx->assoclen, authsize); + + err = alg_ctx->ops.hw_get_result(rk_dev, hard_tag, authsize); + if (err) + goto out_rx; + + CRYPTO_DUMPHEX("hard_tag", hard_tag, authsize); + if (!ctx->is_enc) { + if (!sg_pcopy_to_buffer(alg_ctx->req_src, + sg_nents(alg_ctx->req_src), + user_tag, authsize, + alg_ctx->total + + alg_ctx->assoclen)) { + err = -EINVAL; + goto out_rx; + } + + CRYPTO_DUMPHEX("user_tag", user_tag, authsize); + err = crypto_memneq(user_tag, hard_tag, authsize) ? -EBADMSG : 0; + } else { + if (!sg_pcopy_from_buffer(alg_ctx->req_dst, + sg_nents(alg_ctx->req_dst), + hard_tag, authsize, + alg_ctx->total + + alg_ctx->assoclen)) { + err = -EINVAL; + goto out_rx; + } + } + } else { + rk_iv_copyback(rk_dev); + } } out_rx: return err; @@ -310,10 +355,128 @@ int rk_ablk_start(struct rk_crypto_dev *rk_dev) int rk_skcipher_handle_req(struct rk_crypto_dev *rk_dev, struct skcipher_request *req) { struct rk_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); if (!IS_ALIGNED(req->cryptlen, ctx->algs_ctx.chunk_size) && - !is_no_multi_blocksize(req)) + !is_no_multi_blocksize(algt->mode)) return -EINVAL; else return rk_dev->enqueue(rk_dev, &req->base); } + +int rk_aead_fallback(struct aead_request *req, struct rk_cipher_ctx *ctx, bool encrypt) +{ + int ret; + struct aead_request *subreq = aead_request_ctx(req); + + if (!ctx->fallback_aead) { + CRYPTO_TRACE("fallback_tfm is empty"); + return -EINVAL; + } + + CRYPTO_MSG("use fallback tfm"); + + if (!ctx->fallback_key_inited) { + ret = crypto_aead_setkey(ctx->fallback_aead, ctx->key, ctx->keylen); + if (ret) { + CRYPTO_MSG("fallback crypto_skcipher_setkey err = %d\n", ret); + goto exit; + } + + ctx->fallback_key_inited = true; + } + + aead_request_set_tfm(subreq, ctx->fallback_aead); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + ret = encrypt ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); + +exit: + return ret; +} + +int rk_aead_setkey(struct crypto_aead *cipher, const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct rk_crypto_algt *algt = rk_aead_get_algt(cipher); + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + int ret = -EINVAL; + + CRYPTO_MSG("algo = %x, mode = %x, key_len = %d\n", algt->algo, algt->mode, keylen); + + switch (algt->algo) { + case CIPHER_ALGO_AES: + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + goto error; + + break; + case CIPHER_ALGO_SM4: + if (keylen != SM4_KEY_SIZE) + goto error; + + break; + default: + CRYPTO_TRACE(); + goto error; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + ctx->fallback_key_inited = false; + + return 0; + +error: + return ret; +} + +int rk_aead_start(struct rk_crypto_dev *rk_dev) +{ + struct aead_request *req = aead_request_cast(rk_dev->async_req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm); + struct rk_crypto_algt *algt = rk_aead_get_algt(tfm); + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + unsigned int total = 0, authsize; + unsigned long flags; + int err = 0; + + total = req->cryptlen + req->assoclen; + + authsize = ctx->is_enc ? 0 : crypto_aead_authsize(tfm); + + alg_ctx->total = req->cryptlen - authsize; + alg_ctx->assoclen = req->assoclen; + alg_ctx->sg_src = req->src; + alg_ctx->req_src = req->src; + alg_ctx->src_nents = sg_nents_for_len(req->src, total); + alg_ctx->sg_dst = req->dst; + alg_ctx->req_dst = req->dst; + alg_ctx->dst_nents = sg_nents_for_len(req->dst, total - authsize); + alg_ctx->left_bytes = alg_ctx->total; + + CRYPTO_TRACE("src_nents = %zu, dst_nents = %zu", alg_ctx->src_nents, alg_ctx->dst_nents); + CRYPTO_TRACE("is_enc = %d, authsize = %u, cryptlen = %u, total = %u, assoclen = %u", + ctx->is_enc, authsize, req->cryptlen, alg_ctx->total, alg_ctx->assoclen); + + spin_lock_irqsave(&rk_dev->lock, flags); + alg_ctx->ops.hw_init(rk_dev, algt->algo, algt->mode); + err = rk_set_data_start(rk_dev); + spin_unlock_irqrestore(&rk_dev->lock, flags); + return err; +} + +int rk_aead_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +int rk_aead_handle_req(struct rk_crypto_dev *rk_dev, struct aead_request *req) +{ + return rk_dev->enqueue(rk_dev, &req->base); +} diff --git a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h index 74ac04eef7af..7d47f9719ac8 100644 --- a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h +++ b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h @@ -5,16 +5,23 @@ #ifndef __RK_CRYPTO_SKCIPHER_UTILS_H__ #define __RK_CRYPTO_SKCIPHER_UTILS_H__ +#include #include #include #include "rk_crypto_core.h" #include "rk_crypto_utils.h" +#define RK_MAX_TAG_SIZE 32 + struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm); +struct rk_crypto_algt *rk_aead_get_algt(struct crypto_aead *tfm); + struct rk_alg_ctx *rk_cipher_alg_ctx(struct rk_crypto_dev *rk_dev); +struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev); + int rk_cipher_fallback(struct skcipher_request *req, struct rk_cipher_ctx *ctx, bool encrypt); int rk_cipher_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); @@ -25,5 +32,15 @@ int rk_ablk_start(struct rk_crypto_dev *rk_dev); int rk_skcipher_handle_req(struct rk_crypto_dev *rk_dev, struct skcipher_request *req); +int rk_aead_fallback(struct aead_request *req, struct rk_cipher_ctx *ctx, bool encrypt); + +int rk_aead_setkey(struct crypto_aead *cipher, const u8 *key, unsigned int keylen); + +int rk_aead_start(struct rk_crypto_dev *rk_dev); + +int rk_aead_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize); + +int rk_aead_handle_req(struct rk_crypto_dev *rk_dev, struct aead_request *req); + #endif diff --git a/drivers/crypto/rockchip/rk_crypto_utils.c b/drivers/crypto/rockchip/rk_crypto_utils.c index 0ae2000ed6fb..ded30360a885 100644 --- a/drivers/crypto/rockchip/rk_crypto_utils.c +++ b/drivers/crypto/rockchip/rk_crypto_utils.c @@ -8,6 +8,7 @@ * */ +#include #include #include "rk_crypto_core.h" @@ -174,10 +175,18 @@ int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc) memset(hw_desc, 0x00, sizeof(*hw_desc)); + hw_desc->lli_aad = dma_alloc_coherent(dev, sizeof(struct crypto_lli_desc), + &hw_desc->lli_aad_dma, GFP_KERNEL); + if (!hw_desc->lli_aad) + return -ENOMEM; + ///TODO: cma hw_desc->lli_head = dma_alloc_coherent(dev, lli_len, &hw_desc->lli_head_dma, GFP_KERNEL); - if (!hw_desc->lli_head) + if (!hw_desc->lli_head) { + dma_free_coherent(dev, sizeof(struct crypto_lli_desc), + hw_desc->lli_aad, hw_desc->lli_aad_dma); return -ENOMEM; + } hw_desc->lli_tail = hw_desc->lli_head; hw_desc->total = lli_cnt; @@ -185,9 +194,9 @@ int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc) memset(hw_desc->lli_head, 0x00, lli_len); - CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x", + CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %lx", (unsigned long)hw_desc->dev, lli_len, - (unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma); + (unsigned long)hw_desc->lli_head, (unsigned long)hw_desc->lli_head_dma); return 0; } @@ -197,9 +206,13 @@ void rk_crypto_hw_desc_free(struct rk_hw_desc *hw_desc) if (!hw_desc || !hw_desc->dev || !hw_desc->lli_head) return; - CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x", - (unsigned long)hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc), - (unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma); + CRYPTO_TRACE("dev = %lx, buffer_len = %lu, lli_head = %lx, lli_head_dma = %lx", + (unsigned long)hw_desc->dev, + (unsigned long)hw_desc->total * sizeof(struct crypto_lli_desc), + (unsigned long)hw_desc->lli_head, (unsigned long)hw_desc->lli_head_dma); + + dma_free_coherent(hw_desc->dev, sizeof(struct crypto_lli_desc), + hw_desc->lli_aad, hw_desc->lli_aad_dma); dma_free_coherent(hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc), hw_desc->lli_head, hw_desc->lli_head_dma); diff --git a/drivers/crypto/rockchip/rk_crypto_utils.h b/drivers/crypto/rockchip/rk_crypto_utils.h index c31ba35628d0..21a92fa09077 100644 --- a/drivers/crypto/rockchip/rk_crypto_utils.h +++ b/drivers/crypto/rockchip/rk_crypto_utils.h @@ -26,9 +26,11 @@ struct crypto_lli_desc { struct rk_hw_desc { struct device *dev; + struct crypto_lli_desc *lli_aad; struct crypto_lli_desc *lli_head; struct crypto_lli_desc *lli_tail; dma_addr_t lli_head_dma; + dma_addr_t lli_aad_dma; u32 total; }; diff --git a/drivers/crypto/rockchip/rk_crypto_v2.c b/drivers/crypto/rockchip/rk_crypto_v2.c index 5a37a855fccf..e8f2a0a242a5 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2.c +++ b/drivers/crypto/rockchip/rk_crypto_v2.c @@ -22,6 +22,7 @@ static struct rk_crypto_algt *crypto_v2_algs[] = { &rk_v2_cfb_sm4_alg, /* cfb(sm4) */ &rk_v2_ofb_sm4_alg, /* ofb(sm4) */ &rk_v2_ctr_sm4_alg, /* ctr(sm4) */ + &rk_v2_gcm_sm4_alg, /* gcm(sm4) */ &rk_v2_ecb_aes_alg, /* ecb(aes) */ &rk_v2_cbc_aes_alg, /* cbc(aes) */ @@ -29,6 +30,7 @@ static struct rk_crypto_algt *crypto_v2_algs[] = { &rk_v2_cfb_aes_alg, /* cfb(aes) */ &rk_v2_ofb_aes_alg, /* ofb(aes) */ &rk_v2_ctr_aes_alg, /* ctr(aes) */ + &rk_v2_gcm_aes_alg, /* gcm(aes) */ &rk_v2_ecb_des_alg, /* ecb(des) */ &rk_v2_cbc_des_alg, /* cbc(des) */ diff --git a/drivers/crypto/rockchip/rk_crypto_v2.h b/drivers/crypto/rockchip/rk_crypto_v2.h index 401b166b3359..9e82346d6440 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2.h +++ b/drivers/crypto/rockchip/rk_crypto_v2.h @@ -36,6 +36,7 @@ extern struct rk_crypto_algt rk_v2_xts_sm4_alg; extern struct rk_crypto_algt rk_v2_cfb_sm4_alg; extern struct rk_crypto_algt rk_v2_ofb_sm4_alg; extern struct rk_crypto_algt rk_v2_ctr_sm4_alg; +extern struct rk_crypto_algt rk_v2_gcm_sm4_alg; extern struct rk_crypto_algt rk_v2_ecb_aes_alg; extern struct rk_crypto_algt rk_v2_cbc_aes_alg; @@ -43,6 +44,7 @@ extern struct rk_crypto_algt rk_v2_xts_aes_alg; extern struct rk_crypto_algt rk_v2_cfb_aes_alg; extern struct rk_crypto_algt rk_v2_ofb_aes_alg; extern struct rk_crypto_algt rk_v2_ctr_aes_alg; +extern struct rk_crypto_algt rk_v2_gcm_aes_alg; extern struct rk_crypto_algt rk_v2_ecb_des_alg; extern struct rk_crypto_algt rk_v2_cbc_des_alg; diff --git a/drivers/crypto/rockchip/rk_crypto_v2_reg.h b/drivers/crypto/rockchip/rk_crypto_v2_reg.h index d149265dfab6..a938ce3f69a7 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_reg.h +++ b/drivers/crypto/rockchip/rk_crypto_v2_reg.h @@ -214,6 +214,40 @@ #define CRYPTO_CH7_KEY_3 0x01fc #define CRYPTO_KEY_CHANNEL_NUM 8 +#define CRYPTO_CH0_PC_LEN_0 0x0280 +#define CRYPTO_CH0_PC_LEN_1 0x0284 +#define CRYPTO_CH1_PC_LEN_0 0x0288 +#define CRYPTO_CH1_PC_LEN_1 0x028c +#define CRYPTO_CH2_PC_LEN_0 0x0290 +#define CRYPTO_CH2_PC_LEN_1 0x0294 +#define CRYPTO_CH3_PC_LEN_0 0x0298 +#define CRYPTO_CH3_PC_LEN_1 0x029c +#define CRYPTO_CH4_PC_LEN_0 0x02a0 +#define CRYPTO_CH4_PC_LEN_1 0x02a4 +#define CRYPTO_CH5_PC_LEN_0 0x02a8 +#define CRYPTO_CH5_PC_LEN_1 0x02ac +#define CRYPTO_CH6_PC_LEN_0 0x02b0 +#define CRYPTO_CH6_PC_LEN_1 0x02b4 +#define CRYPTO_CH7_PC_LEN_0 0x02b8 +#define CRYPTO_CH7_PC_LEN_1 0x02bc + +#define CRYPTO_CH0_AAD_LEN_0 0x02c0 +#define CRYPTO_CH0_AAD_LEN_1 0x02c4 +#define CRYPTO_CH1_AAD_LEN_0 0x02c8 +#define CRYPTO_CH1_AAD_LEN_1 0x02cc +#define CRYPTO_CH2_AAD_LEN_0 0x02d0 +#define CRYPTO_CH2_AAD_LEN_1 0x02d4 +#define CRYPTO_CH3_AAD_LEN_0 0x02d8 +#define CRYPTO_CH3_AAD_LEN_1 0x02dc +#define CRYPTO_CH4_AAD_LEN_0 0x02e0 +#define CRYPTO_CH4_AAD_LEN_1 0x02e4 +#define CRYPTO_CH5_AAD_LEN_0 0x02e8 +#define CRYPTO_CH5_AAD_LEN_1 0x02ec +#define CRYPTO_CH6_AAD_LEN_0 0x02f0 +#define CRYPTO_CH6_AAD_LEN_1 0x02f4 +#define CRYPTO_CH7_AAD_LEN_0 0x02f8 +#define CRYPTO_CH7_AAD_LEN_1 0x02fc + #define CRYPTO_CH0_IV_LEN_0 0x0300 #define CRYPTO_CH1_IV_LEN_0 0x0304 #define CRYPTO_CH2_IV_LEN_0 0x0308 @@ -223,6 +257,11 @@ #define CRYPTO_CH6_IV_LEN_0 0x0318 #define CRYPTO_CH7_IV_LEN_0 0x031c +#define CRYPTO_CH0_TAG_0 0x0320 +#define CRYPTO_CH0_TAG_1 0x0324 +#define CRYPTO_CH0_TAG_2 0x0328 +#define CRYPTO_CH0_TAG_3 0x032c + #define CRYPTO_HASH_DOUT_0 0x03a0 #define CRYPTO_HASH_DOUT_1 0x03a4 #define CRYPTO_HASH_DOUT_2 0x03a8 @@ -240,6 +279,9 @@ #define CRYPTO_HASH_DOUT_14 0x03d8 #define CRYPTO_HASH_DOUT_15 0x03dc +#define CRYPTO_TAG_VALID 0x03e0 +#define CRYPTO_CH0_TAG_VALID BIT(0) + #define CRYPTO_HASH_VALID 0x03e4 #define CRYPTO_HASH_IS_VALID BIT(0) diff --git a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c index e929ff66477d..cce8e86228bc 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c @@ -9,15 +9,20 @@ * Some ideas are from marvell-cesa.c and s5p-sss.c driver. */ +#include #include #include #include #include "rk_crypto_core.h" +#include "rk_crypto_utils.h" #include "rk_crypto_skcipher_utils.h" #include "rk_crypto_v2.h" #include "rk_crypto_v2_reg.h" +#define RK_POLL_PERIOD_US 100 +#define RK_POLL_TIMEOUT_US 50000 + static const u32 cipher_algo2bc[] = { [CIPHER_ALGO_DES] = CRYPTO_BC_DES, [CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES, @@ -32,6 +37,7 @@ static const u32 cipher_mode2bc[] = { [CIPHER_MODE_OFB] = CRYPTO_BC_OFB, [CIPHER_MODE_CTR] = CRYPTO_BC_CTR, [CIPHER_MODE_XTS] = CRYPTO_BC_XTS, + [CIPHER_MODE_GCM] = CRYPTO_BC_GCM, }; static int rk_crypto_irq_handle(int irq, void *dev_id) @@ -76,6 +82,26 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) return 0; } +static inline void set_pc_len_reg(struct rk_crypto_dev *rk_dev, u64 pc_len) +{ + u32 chn_base = CRYPTO_CH0_PC_LEN_0; + + CRYPTO_TRACE("PC length = %lu\n", (unsigned long)pc_len); + + CRYPTO_WRITE(rk_dev, chn_base, pc_len & 0xffffffff); + CRYPTO_WRITE(rk_dev, chn_base + 4, pc_len >> 32); +} + +static inline void set_aad_len_reg(struct rk_crypto_dev *rk_dev, u64 aad_len) +{ + u32 chn_base = CRYPTO_CH0_AAD_LEN_0; + + CRYPTO_TRACE("AAD length = %lu\n", (unsigned long)aad_len); + + CRYPTO_WRITE(rk_dev, chn_base, aad_len & 0xffffffff); + CRYPTO_WRITE(rk_dev, chn_base + 4, aad_len >> 32); +} + static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len) { if (!iv || iv_len == 0) @@ -100,6 +126,29 @@ static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key, rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len); } +static int get_tag_reg(struct rk_crypto_dev *rk_dev, u8 *tag, u32 tag_len) +{ + int ret; + u32 reg_ctrl = 0; + + CRYPTO_TRACE("tag_len = %u", tag_len); + + if (tag_len > RK_MAX_TAG_SIZE) + return -EINVAL; + + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_TAG_VALID, + reg_ctrl, + reg_ctrl & CRYPTO_CH0_TAG_VALID, + RK_POLL_PERIOD_US, + RK_POLL_TIMEOUT_US); + if (ret) + goto exit; + + rk_crypto_read_regs(rk_dev, CRYPTO_CH0_TAG_0, tag, tag_len); +exit: + return ret; +} + static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len) { if (algt->algo != CIPHER_ALGO_AES) @@ -230,8 +279,10 @@ static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt) ctx->mode |= CRYPTO_BC_256_bit_key; } - if (req->iv) - memcpy(ctx->iv, req->iv, crypto_skcipher_ivsize(tfm)); + ctx->iv_len = crypto_skcipher_ivsize(tfm); + + memset(ctx->iv, 0x00, sizeof(ctx->iv)); + memcpy(ctx->iv, req->iv, ctx->iv_len); ctx->is_enc = encrypt; @@ -251,18 +302,12 @@ static int rk_cipher_decrypt(struct skcipher_request *req) static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode) { - struct skcipher_request *req = - skcipher_request_cast(rk_dev->async_req); - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); - u32 ivsize; + struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); rk_cipher_reset(rk_dev); CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000); - ivsize = crypto_skcipher_ivsize(cipher); - if (mode == CIPHER_MODE_XTS) { uint32_t tmp_len = ctx->keylen / 2; @@ -273,7 +318,7 @@ static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode) } if (mode != CIPHER_MODE_ECB) - set_iv_reg(rk_dev, req->iv, ivsize); + set_iv_reg(rk_dev, ctx->iv, ctx->iv_len); ctx->mode |= CRYPTO_BC_ENABLE; @@ -293,7 +338,7 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); - struct crypto_lli_desc *lli_head, *lli_tail; + struct crypto_lli_desc *lli_head, *lli_tail, *lli_aad; u32 calc_len = alg_ctx->count; u32 start_flag = CRYPTO_DMA_START; int ret; @@ -309,26 +354,50 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) lli_head = hw_info->hw_desc.lli_head; lli_tail = hw_info->hw_desc.lli_tail; + lli_aad = hw_info->hw_desc.lli_aad; /* * the data length is not aligned will use addr_vir to calculate, * so crypto v2 could round up data length to chunk_size */ - if (is_calc_need_round_up(req)) + if (!alg_ctx->is_aead && is_calc_need_round_up(req)) calc_len = round_up(calc_len, alg_ctx->chunk_size); + CRYPTO_TRACE("calc_len = %u, cryptlen = %u, assoclen= %u, is_aead = %d", + calc_len, alg_ctx->total, alg_ctx->assoclen, alg_ctx->is_aead); + lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START; - lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; + lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; lli_tail->user_define |= LLI_USER_STRING_LAST; - lli_tail->src_len += (calc_len - alg_ctx->count); - lli_tail->dst_len += (calc_len - alg_ctx->count); + lli_tail->src_len += (calc_len - alg_ctx->count); + lli_tail->dst_len += (calc_len - alg_ctx->count); + + if (alg_ctx->is_aead) { + lli_aad->src_addr = alg_ctx->addr_aad_in; + lli_aad->src_len = alg_ctx->assoclen; + lli_aad->user_define = LLI_USER_CIPHER_START | + LLI_USER_STRING_START | + LLI_USER_STRING_LAST | + LLI_USER_STRING_AAD; + lli_aad->next_addr = hw_info->hw_desc.lli_head_dma; + + /* clear cipher start */ + lli_head->user_define &= (~((u32)LLI_USER_CIPHER_START)); + + set_pc_len_reg(rk_dev, alg_ctx->total); + set_aad_len_reg(rk_dev, alg_ctx->assoclen); + } rk_crypto_dump_hw_desc(&hw_info->hw_desc); dma_wmb(); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); + if (alg_ctx->is_aead) + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_aad_dma); + else + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); return 0; @@ -396,6 +465,151 @@ static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm) ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name); } +static int rk_aead_init_tfm(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct rk_crypto_algt *algt = + container_of(alg, struct rk_crypto_algt, alg.aead); + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base); + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct rk_crypto_dev *rk_dev = algt->rk_dev; + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + + CRYPTO_TRACE(); + + if (!rk_dev->request_crypto) + return -EFAULT; + + rk_dev->request_crypto(rk_dev, alg_name); + + alg_ctx->align_size = 64; + alg_ctx->chunk_size = crypto_aead_chunksize(tfm); + + alg_ctx->ops.start = rk_aead_start; + alg_ctx->ops.update = rk_ablk_rx; + alg_ctx->ops.complete = rk_crypto_complete; + alg_ctx->ops.irq_handle = rk_crypto_irq_handle; + + alg_ctx->ops.hw_init = rk_ablk_hw_init; + alg_ctx->ops.hw_dma_start = crypto_dma_start; + alg_ctx->ops.hw_write_iv = set_iv_reg; + alg_ctx->ops.hw_get_result = get_tag_reg; + + ctx->rk_dev = rk_dev; + alg_ctx->is_aead = 1; + + if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name); + ctx->fallback_aead = + crypto_alloc_aead(alg_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_aead)) { + dev_err(rk_dev->dev, + "Load fallback driver %s err: %ld.\n", + alg_name, PTR_ERR(ctx->fallback_aead)); + ctx->fallback_aead = NULL; + } + + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->fallback_aead)); + } + + return 0; +} + +static void rk_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + CRYPTO_TRACE(); + + if (ctx->fallback_aead) { + CRYPTO_MSG("free fallback tfm"); + crypto_free_aead(ctx->fallback_aead); + } + + ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(&tfm->base)); +} + +static int rk_aead_crypt(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm); + struct rk_crypto_algt *algt = rk_aead_get_algt(tfm); + struct scatterlist *sg_src, *sg_dst; + struct scatterlist src[2], dst[2]; + u64 data_len; + bool aligned; + int ret = -EINVAL; + + CRYPTO_TRACE("%s cryptlen = %u, assoclen = %u", + encrypt ? "encrypt" : "decrypt", + req->cryptlen, req->assoclen); + + data_len = encrypt ? req->cryptlen : (req->cryptlen - crypto_aead_authsize(tfm)); + + if (req->assoclen == 0 || + req->cryptlen == 0 || + data_len == 0 || + is_force_fallback(algt, ctx->keylen)) + return rk_aead_fallback(req, ctx, encrypt); + + /* point sg_src and sg_dst skip assoc data */ + sg_src = scatterwalk_ffwd(src, req->src, req->assoclen); + sg_dst = (req->src == req->dst) ? sg_src : scatterwalk_ffwd(dst, req->dst, req->assoclen); + + aligned = rk_crypto_check_align(sg_src, sg_nents_for_len(sg_src, data_len), + sg_dst, sg_nents_for_len(sg_dst, data_len), + 64); + + if (sg_nents_for_len(sg_src, data_len) > RK_DEFAULT_LLI_CNT || + sg_nents_for_len(sg_dst, data_len) > RK_DEFAULT_LLI_CNT) + return rk_aead_fallback(req, ctx, encrypt); + + if (!aligned) { + if (req->assoclen > ctx->rk_dev->aad_max || + data_len > ctx->rk_dev->vir_max) + return rk_aead_fallback(req, ctx, encrypt); + } + + ctx->mode = cipher_algo2bc[algt->algo] | + cipher_mode2bc[algt->mode]; + if (!encrypt) + ctx->mode |= CRYPTO_BC_DECRYPT; + + if (algt->algo == CIPHER_ALGO_AES) { + if (ctx->keylen == AES_KEYSIZE_128) + ctx->mode |= CRYPTO_BC_128_bit_key; + else if (ctx->keylen == AES_KEYSIZE_192) + ctx->mode |= CRYPTO_BC_192_bit_key; + else if (ctx->keylen == AES_KEYSIZE_256) + ctx->mode |= CRYPTO_BC_256_bit_key; + } + + ctx->iv_len = crypto_aead_ivsize(tfm); + + memset(ctx->iv, 0x00, sizeof(ctx->iv)); + memcpy(ctx->iv, req->iv, ctx->iv_len); + + ctx->is_enc = encrypt; + + CRYPTO_MSG("ctx->mode = %x\n", ctx->mode); + ret = rk_aead_handle_req(ctx->rk_dev, req); + + return ret; +} + +static int rk_aead_encrypt(struct aead_request *req) +{ + return rk_aead_crypt(req, true); +} + +static int rk_aead_decrypt(struct aead_request *req) +{ + return rk_aead_crypt(req, false); +} + struct rk_crypto_algt rk_v2_ecb_sm4_alg = RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk); @@ -414,6 +628,9 @@ struct rk_crypto_algt rk_v2_ofb_sm4_alg = struct rk_crypto_algt rk_v2_ctr_sm4_alg = RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk); +struct rk_crypto_algt rk_v2_gcm_sm4_alg = + RK_AEAD_ALGO_INIT(SM4, GCM, gcm(sm4), gcm-sm4-rk); + struct rk_crypto_algt rk_v2_ecb_aes_alg = RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk); @@ -432,6 +649,9 @@ struct rk_crypto_algt rk_v2_ofb_aes_alg = struct rk_crypto_algt rk_v2_ctr_aes_alg = RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk); +struct rk_crypto_algt rk_v2_gcm_aes_alg = + RK_AEAD_ALGO_INIT(AES, GCM, gcm(aes), gcm-aes-rk); + struct rk_crypto_algt rk_v2_ecb_des_alg = RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk); diff --git a/drivers/crypto/rockchip/rk_crypto_v3.c b/drivers/crypto/rockchip/rk_crypto_v3.c index b4f339019fba..7cd728599d88 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3.c +++ b/drivers/crypto/rockchip/rk_crypto_v3.c @@ -52,46 +52,48 @@ static const char * const crypto_v3_rsts[] = { }; static struct rk_crypto_algt *crypto_v3_algs[] = { - &rk_v3_ecb_sm4_alg, /* ecb(sm4) */ - &rk_v3_cbc_sm4_alg, /* cbc(sm4) */ - &rk_v3_xts_sm4_alg, /* xts(sm4) */ - &rk_v3_cfb_sm4_alg, /* cfb(sm4) */ - &rk_v3_ofb_sm4_alg, /* ofb(sm4) */ - &rk_v3_ctr_sm4_alg, /* ctr(sm4) */ + &rk_v3_ecb_sm4_alg, /* ecb(sm4) */ + &rk_v3_cbc_sm4_alg, /* cbc(sm4) */ + &rk_v3_xts_sm4_alg, /* xts(sm4) */ + &rk_v3_cfb_sm4_alg, /* cfb(sm4) */ + &rk_v3_ofb_sm4_alg, /* ofb(sm4) */ + &rk_v3_ctr_sm4_alg, /* ctr(sm4) */ + &rk_v3_gcm_sm4_alg, /* ctr(sm4) */ - &rk_v3_ecb_aes_alg, /* ecb(aes) */ - &rk_v3_cbc_aes_alg, /* cbc(aes) */ - &rk_v3_xts_aes_alg, /* xts(aes) */ - &rk_v3_cfb_aes_alg, /* cfb(aes) */ - &rk_v3_ofb_aes_alg, /* ofb(aes) */ - &rk_v3_ctr_aes_alg, /* ctr(aes) */ + &rk_v3_ecb_aes_alg, /* ecb(aes) */ + &rk_v3_cbc_aes_alg, /* cbc(aes) */ + &rk_v3_xts_aes_alg, /* xts(aes) */ + &rk_v3_cfb_aes_alg, /* cfb(aes) */ + &rk_v3_ofb_aes_alg, /* ofb(aes) */ + &rk_v3_ctr_aes_alg, /* ctr(aes) */ + &rk_v3_gcm_aes_alg, /* gcm(aes) */ - &rk_v3_ecb_des_alg, /* ecb(des) */ - &rk_v3_cbc_des_alg, /* cbc(des) */ - &rk_v3_cfb_des_alg, /* cfb(des) */ - &rk_v3_ofb_des_alg, /* ofb(des) */ + &rk_v3_ecb_des_alg, /* ecb(des) */ + &rk_v3_cbc_des_alg, /* cbc(des) */ + &rk_v3_cfb_des_alg, /* cfb(des) */ + &rk_v3_ofb_des_alg, /* ofb(des) */ - &rk_v3_ecb_des3_ede_alg, /* ecb(des3_ede) */ - &rk_v3_cbc_des3_ede_alg, /* cbc(des3_ede) */ - &rk_v3_cfb_des3_ede_alg, /* cfb(des3_ede) */ - &rk_v3_ofb_des3_ede_alg, /* ofb(des3_ede) */ + &rk_v3_ecb_des3_ede_alg, /* ecb(des3_ede) */ + &rk_v3_cbc_des3_ede_alg, /* cbc(des3_ede) */ + &rk_v3_cfb_des3_ede_alg, /* cfb(des3_ede) */ + &rk_v3_ofb_des3_ede_alg, /* ofb(des3_ede) */ - &rk_v3_ahash_sha1, /* sha1 */ - &rk_v3_ahash_sha224, /* sha224 */ - &rk_v3_ahash_sha256, /* sha256 */ - &rk_v3_ahash_sha384, /* sha384 */ - &rk_v3_ahash_sha512, /* sha512 */ - &rk_v3_ahash_md5, /* md5 */ - &rk_v3_ahash_sm3, /* sm3 */ + &rk_v3_ahash_sha1, /* sha1 */ + &rk_v3_ahash_sha224, /* sha224 */ + &rk_v3_ahash_sha256, /* sha256 */ + &rk_v3_ahash_sha384, /* sha384 */ + &rk_v3_ahash_sha512, /* sha512 */ + &rk_v3_ahash_md5, /* md5 */ + &rk_v3_ahash_sm3, /* sm3 */ - &rk_v3_hmac_sha1, /* hmac(sha1) */ - &rk_v3_hmac_sha256, /* hmac(sha256) */ - &rk_v3_hmac_sha512, /* hmac(sha512) */ - &rk_v3_hmac_md5, /* hmac(md5) */ - &rk_v3_hmac_sm3, /* hmac(sm3) */ + &rk_v3_hmac_sha1, /* hmac(sha1) */ + &rk_v3_hmac_sha256, /* hmac(sha256) */ + &rk_v3_hmac_sha512, /* hmac(sha512) */ + &rk_v3_hmac_md5, /* hmac(md5) */ + &rk_v3_hmac_sm3, /* hmac(sm3) */ - /* Shared v2 version implementation */ - &rk_v2_asym_rsa, /* rsa */ + /* Shared v2 version implementation */ + &rk_v2_asym_rsa, /* rsa */ }; static bool rk_is_cipher_support(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode, u32 key_len) @@ -199,7 +201,7 @@ struct rk_crypto_algt **rk_hw_crypto_v3_get_algts(uint32_t *num) bool rk_hw_crypto_v3_algo_valid(struct rk_crypto_dev *rk_dev, struct rk_crypto_algt *aglt) { - if (aglt->type == ALG_TYPE_CIPHER) { + if (aglt->type == ALG_TYPE_CIPHER || aglt->type == ALG_TYPE_AEAD) { CRYPTO_TRACE("CIPHER"); return rk_is_cipher_support(rk_dev, aglt->algo, aglt->mode, 0); } else if (aglt->type == ALG_TYPE_HASH || aglt->type == ALG_TYPE_HMAC) { diff --git a/drivers/crypto/rockchip/rk_crypto_v3.h b/drivers/crypto/rockchip/rk_crypto_v3.h index e5ac6d2931ff..a4b181416300 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3.h +++ b/drivers/crypto/rockchip/rk_crypto_v3.h @@ -36,6 +36,7 @@ extern struct rk_crypto_algt rk_v3_xts_sm4_alg; extern struct rk_crypto_algt rk_v3_cfb_sm4_alg; extern struct rk_crypto_algt rk_v3_ofb_sm4_alg; extern struct rk_crypto_algt rk_v3_ctr_sm4_alg; +extern struct rk_crypto_algt rk_v3_gcm_sm4_alg; extern struct rk_crypto_algt rk_v3_ecb_aes_alg; extern struct rk_crypto_algt rk_v3_cbc_aes_alg; @@ -43,6 +44,7 @@ extern struct rk_crypto_algt rk_v3_xts_aes_alg; extern struct rk_crypto_algt rk_v3_cfb_aes_alg; extern struct rk_crypto_algt rk_v3_ofb_aes_alg; extern struct rk_crypto_algt rk_v3_ctr_aes_alg; +extern struct rk_crypto_algt rk_v3_gcm_aes_alg; extern struct rk_crypto_algt rk_v3_ecb_des_alg; extern struct rk_crypto_algt rk_v3_cbc_des_alg; diff --git a/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c index 21a71da4082e..06f7f74a8694 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c @@ -8,15 +8,20 @@ * */ +#include #include #include #include #include "rk_crypto_core.h" +#include "rk_crypto_utils.h" #include "rk_crypto_skcipher_utils.h" #include "rk_crypto_v3.h" #include "rk_crypto_v3_reg.h" +#define RK_POLL_PERIOD_US 100 +#define RK_POLL_TIMEOUT_US 50000 + static const u32 cipher_algo2bc[] = { [CIPHER_ALGO_DES] = CRYPTO_BC_DES, [CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES, @@ -31,6 +36,7 @@ static const u32 cipher_mode2bc[] = { [CIPHER_MODE_OFB] = CRYPTO_BC_OFB, [CIPHER_MODE_CTR] = CRYPTO_BC_CTR, [CIPHER_MODE_XTS] = CRYPTO_BC_XTS, + [CIPHER_MODE_GCM] = CRYPTO_BC_GCM, }; static int rk_crypto_irq_handle(int irq, void *dev_id) @@ -75,6 +81,26 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) return 0; } +static inline void set_pc_len_reg(struct rk_crypto_dev *rk_dev, u64 pc_len) +{ + u32 chn_base = CRYPTO_CH0_PC_LEN_0; + + CRYPTO_TRACE("PC length = %lu\n", (unsigned long)pc_len); + + CRYPTO_WRITE(rk_dev, chn_base, pc_len & 0xffffffff); + CRYPTO_WRITE(rk_dev, chn_base + 4, pc_len >> 32); +} + +static inline void set_aad_len_reg(struct rk_crypto_dev *rk_dev, u64 aad_len) +{ + u32 chn_base = CRYPTO_CH0_AAD_LEN_0; + + CRYPTO_TRACE("AAD length = %lu\n", (unsigned long)aad_len); + + CRYPTO_WRITE(rk_dev, chn_base, aad_len & 0xffffffff); + CRYPTO_WRITE(rk_dev, chn_base + 4, aad_len >> 32); +} + static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len) { if (!iv || iv_len == 0) @@ -99,6 +125,29 @@ static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key, rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len); } +static int get_tag_reg(struct rk_crypto_dev *rk_dev, u8 *tag, u32 tag_len) +{ + int ret; + u32 reg_ctrl = 0; + + CRYPTO_TRACE("tag_len = %u", tag_len); + + if (tag_len > RK_MAX_TAG_SIZE) + return -EINVAL; + + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_TAG_VALID, + reg_ctrl, + reg_ctrl & CRYPTO_CH0_TAG_VALID, + RK_POLL_PERIOD_US, + RK_POLL_TIMEOUT_US); + if (ret) + goto exit; + + rk_crypto_read_regs(rk_dev, CRYPTO_CH0_TAG_0, tag, tag_len); +exit: + return ret; +} + static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len) { if (algt->algo != CIPHER_ALGO_AES) @@ -120,8 +169,8 @@ static bool is_calc_need_round_up(struct skcipher_request *req) struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); return (algt->mode == CIPHER_MODE_CFB || - algt->mode == CIPHER_MODE_OFB || - algt->mode == CIPHER_MODE_CTR) ? true : false; + algt->mode == CIPHER_MODE_OFB || + algt->mode == CIPHER_MODE_CTR) ? true : false; } static void rk_cipher_reset(struct rk_crypto_dev *rk_dev) @@ -229,8 +278,10 @@ static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt) ctx->mode |= CRYPTO_BC_256_bit_key; } - if (req->iv) - memcpy(ctx->iv, req->iv, crypto_skcipher_ivsize(tfm)); + ctx->iv_len = crypto_skcipher_ivsize(tfm); + + memset(ctx->iv, 0x00, sizeof(ctx->iv)); + memcpy(ctx->iv, req->iv, ctx->iv_len); ctx->is_enc = encrypt; @@ -250,18 +301,12 @@ static int rk_cipher_decrypt(struct skcipher_request *req) static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode) { - struct skcipher_request *req = - skcipher_request_cast(rk_dev->async_req); - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); - u32 ivsize; + struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); rk_cipher_reset(rk_dev); CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000); - ivsize = crypto_skcipher_ivsize(cipher); - if (mode == CIPHER_MODE_XTS) { uint32_t tmp_len = ctx->keylen / 2; @@ -272,7 +317,7 @@ static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode) } if (mode != CIPHER_MODE_ECB) - set_iv_reg(rk_dev, req->iv, ivsize); + set_iv_reg(rk_dev, ctx->iv, ctx->iv_len); ctx->mode |= CRYPTO_BC_ENABLE; @@ -292,7 +337,7 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); - struct crypto_lli_desc *lli_head, *lli_tail; + struct crypto_lli_desc *lli_head, *lli_tail, *lli_aad; u32 calc_len = alg_ctx->count; u32 start_flag = CRYPTO_DMA_START; int ret; @@ -308,26 +353,50 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) lli_head = hw_info->hw_desc.lli_head; lli_tail = hw_info->hw_desc.lli_tail; + lli_aad = hw_info->hw_desc.lli_aad; /* * the data length is not aligned will use addr_vir to calculate, * so crypto v2 could round up data length to chunk_size */ - if (is_calc_need_round_up(req)) + if (!alg_ctx->is_aead && is_calc_need_round_up(req)) calc_len = round_up(calc_len, alg_ctx->chunk_size); + CRYPTO_TRACE("calc_len = %u, cryptlen = %u, assoclen= %u, is_aead = %d", + calc_len, alg_ctx->total, alg_ctx->assoclen, alg_ctx->is_aead); + lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START; - lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; + lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; lli_tail->user_define |= LLI_USER_STRING_LAST; - lli_tail->src_len += (calc_len - alg_ctx->count); - lli_tail->dst_len += (calc_len - alg_ctx->count); + lli_tail->src_len += (calc_len - alg_ctx->count); + lli_tail->dst_len += (calc_len - alg_ctx->count); + + if (alg_ctx->is_aead) { + lli_aad->src_addr = alg_ctx->addr_aad_in; + lli_aad->src_len = alg_ctx->assoclen; + lli_aad->user_define = LLI_USER_CIPHER_START | + LLI_USER_STRING_START | + LLI_USER_STRING_LAST | + LLI_USER_STRING_AAD; + lli_aad->next_addr = hw_info->hw_desc.lli_head_dma; + + /* clear cipher start */ + lli_head->user_define &= (~((u32)LLI_USER_CIPHER_START)); + + set_pc_len_reg(rk_dev, alg_ctx->total); + set_aad_len_reg(rk_dev, alg_ctx->assoclen); + } rk_crypto_dump_hw_desc(&hw_info->hw_desc); dma_wmb(); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); + if (alg_ctx->is_aead) + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_aad_dma); + else + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); return 0; @@ -395,6 +464,151 @@ static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm) ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name); } +static int rk_aead_init_tfm(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct rk_crypto_algt *algt = + container_of(alg, struct rk_crypto_algt, alg.aead); + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base); + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct rk_crypto_dev *rk_dev = algt->rk_dev; + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + + CRYPTO_TRACE(); + + if (!rk_dev->request_crypto) + return -EFAULT; + + rk_dev->request_crypto(rk_dev, alg_name); + + alg_ctx->align_size = 64; + alg_ctx->chunk_size = crypto_aead_chunksize(tfm); + + alg_ctx->ops.start = rk_aead_start; + alg_ctx->ops.update = rk_ablk_rx; + alg_ctx->ops.complete = rk_crypto_complete; + alg_ctx->ops.irq_handle = rk_crypto_irq_handle; + + alg_ctx->ops.hw_init = rk_ablk_hw_init; + alg_ctx->ops.hw_dma_start = crypto_dma_start; + alg_ctx->ops.hw_write_iv = set_iv_reg; + alg_ctx->ops.hw_get_result = get_tag_reg; + + ctx->rk_dev = rk_dev; + alg_ctx->is_aead = 1; + + if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name); + ctx->fallback_aead = + crypto_alloc_aead(alg_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_aead)) { + dev_err(rk_dev->dev, + "Load fallback driver %s err: %ld.\n", + alg_name, PTR_ERR(ctx->fallback_aead)); + ctx->fallback_aead = NULL; + } + + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->fallback_aead)); + } + + return 0; +} + +static void rk_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + CRYPTO_TRACE(); + + if (ctx->fallback_aead) { + CRYPTO_MSG("free fallback tfm"); + crypto_free_aead(ctx->fallback_aead); + } + + ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(&tfm->base)); +} + +static int rk_aead_crypt(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm); + struct rk_crypto_algt *algt = rk_aead_get_algt(tfm); + struct scatterlist *sg_src, *sg_dst; + struct scatterlist src[2], dst[2]; + u64 data_len; + bool aligned; + int ret = -EINVAL; + + CRYPTO_TRACE("%s cryptlen = %u, assoclen = %u", + encrypt ? "encrypt" : "decrypt", + req->cryptlen, req->assoclen); + + data_len = encrypt ? req->cryptlen : (req->cryptlen - crypto_aead_authsize(tfm)); + + if (req->assoclen == 0 || + req->cryptlen == 0 || + data_len == 0 || + is_force_fallback(algt, ctx->keylen)) + return rk_aead_fallback(req, ctx, encrypt); + + /* point sg_src and sg_dst skip assoc data */ + sg_src = scatterwalk_ffwd(src, req->src, req->assoclen); + sg_dst = (req->src == req->dst) ? sg_src : scatterwalk_ffwd(dst, req->dst, req->assoclen); + + aligned = rk_crypto_check_align(sg_src, sg_nents_for_len(sg_src, data_len), + sg_dst, sg_nents_for_len(sg_dst, data_len), + 64); + + if (sg_nents_for_len(sg_src, data_len) > RK_DEFAULT_LLI_CNT || + sg_nents_for_len(sg_dst, data_len) > RK_DEFAULT_LLI_CNT) + return rk_aead_fallback(req, ctx, encrypt); + + if (!aligned) { + if (req->assoclen > ctx->rk_dev->aad_max || + data_len > ctx->rk_dev->vir_max) + return rk_aead_fallback(req, ctx, encrypt); + } + + ctx->mode = cipher_algo2bc[algt->algo] | + cipher_mode2bc[algt->mode]; + if (!encrypt) + ctx->mode |= CRYPTO_BC_DECRYPT; + + if (algt->algo == CIPHER_ALGO_AES) { + if (ctx->keylen == AES_KEYSIZE_128) + ctx->mode |= CRYPTO_BC_128_bit_key; + else if (ctx->keylen == AES_KEYSIZE_192) + ctx->mode |= CRYPTO_BC_192_bit_key; + else if (ctx->keylen == AES_KEYSIZE_256) + ctx->mode |= CRYPTO_BC_256_bit_key; + } + + ctx->iv_len = crypto_aead_ivsize(tfm); + + memset(ctx->iv, 0x00, sizeof(ctx->iv)); + memcpy(ctx->iv, req->iv, ctx->iv_len); + + ctx->is_enc = encrypt; + + CRYPTO_MSG("ctx->mode = %x\n", ctx->mode); + ret = rk_aead_handle_req(ctx->rk_dev, req); + + return ret; +} + +static int rk_aead_encrypt(struct aead_request *req) +{ + return rk_aead_crypt(req, true); +} + +static int rk_aead_decrypt(struct aead_request *req) +{ + return rk_aead_crypt(req, false); +} + struct rk_crypto_algt rk_v3_ecb_sm4_alg = RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk); @@ -413,6 +627,9 @@ struct rk_crypto_algt rk_v3_ofb_sm4_alg = struct rk_crypto_algt rk_v3_ctr_sm4_alg = RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk); +struct rk_crypto_algt rk_v3_gcm_sm4_alg = + RK_AEAD_ALGO_INIT(SM4, GCM, gcm(sm4), gcm-sm4-rk); + struct rk_crypto_algt rk_v3_ecb_aes_alg = RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk); @@ -431,6 +648,9 @@ struct rk_crypto_algt rk_v3_ofb_aes_alg = struct rk_crypto_algt rk_v3_ctr_aes_alg = RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk); +struct rk_crypto_algt rk_v3_gcm_aes_alg = + RK_AEAD_ALGO_INIT(AES, GCM, gcm(aes), gcm-aes-rk); + struct rk_crypto_algt rk_v3_ecb_des_alg = RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk);